Build an AI Coding Agent That Ships Real Pull Requests: Architecture, Guardrails and Cost Per PR (2026)

import { createHash } from "node:crypto";

type Plan = {
  files: string[];          // declared file set - enforced, not suggested
  testsMustPass: string[];
  testsToWrite: string[];
  turnBudget: number;       // <= MAX_TURNS
};

type TestResult = { passed: boolean; failures: string[]; durationMs: number };

const MAX_TURNS = 12;
const MAX_USD = 2.5;
const NO_PROGRESS_LIMIT = 3;

export type LoopOutcome =
  | { status: "candidate"; diff: string; turns: number; usd: number }
  | { status: "abandoned"; reason: string; turns: number; usd: number };

export async function editLoop(
  plan: Plan,
  sandbox: Sandbox,
  model: Model,
  journal: Journal,
): Promise<LoopOutcome> {
  let usd = 0;
  let seen = new Map<string, number>();
  const budget = Math.min(plan.turnBudget, MAX_TURNS);

  for (let turn = 1; turn <= budget; turn++) {
    if (usd >= MAX_USD) {
      return { status: "abandoned", reason: "usd_budget", turns: turn - 1, usd };
    }

    const context = await sandbox.readFiles(plan.files);
    const step = await model.propose({ plan, context, turn });
    usd += step.usd;

    // Enforce the declared file set. Scope creep is a stop, not a warning.
    const touched = filesInPatch(step.patch);
    const outside = touched.filter((f) => !plan.files.includes(f));
    if (outside.length > 0) {
      await journal.step(turn, "scope_violation", { outside, usd });
      return { status: "abandoned", reason: "scope_violation", turns: turn, usd };
    }

    const applied = await sandbox.applyPatch(step.patch);
    if (!applied.ok) {
      await journal.step(turn, "patch_rejected", { error: applied.error, usd });
      continue; // a rejected hunk is information, not a crash
    }

    const result: TestResult = await sandbox.runTests(plan.testsMustPass);
    const diff = await sandbox.diff();
    await journal.step(turn, "tested", {
      passed: result.passed,
      failures: result.failures.length,
      usd,
    });

    if (result.passed) {
      return { status: "candidate", diff, turns: turn, usd };
    }

    // No-progress detection: same diff shape AND same failure signature.
    const fp = fingerprint(diff, result.failures);
    const n = (seen.get(fp) ?? 0) + 1;
    seen.set(fp, n);
    if (n >= NO_PROGRESS_LIMIT) {
      return { status: "abandoned", reason: "no_progress", turns: turn, usd };
    }
  }

  return { status: "abandoned", reason: "turn_budget", turns: budget, usd };
}

function fingerprint(diff: string, failures: string[]): string {
  const normalisedDiff = diff.replace(/\s+/g, " ").trim();
  const normalisedFailures = failures.map((f) => f.split(":")[0]).sort().join("|");
  return createHash("sha256")
    .update(normalisedDiff + "::" + normalisedFailures)
    .digest("hex");
}

1. 1
   Stop 1 · Intake rejects the task$0.00

   No acceptance criteria, no reproduction, or a file allowlist that spans more than eight files. The task goes back to a human with a specific reason. This is the cheapest possible failure and the most under-used.

2. 2
   Stop 2 · Grounding finds nothing$0.0004

   No symbol match and no chunk above the relevance floor. The agent should say I cannot locate the code this describes rather than guess a module. Refusal is a feature.

3. 3
   Stop 3 · Scope violation~$0.15 at turn 3

   The loop tries to patch a file outside the declared set. Terminate at that turn, journal the attempted files, and surface them — a repeated scope violation on the same task usually means the plan was wrong, not the agent.

4. 4
   Stop 4 · No progress~$0.31 at turn 6

   Three consecutive turns with the same diff-plus-failure fingerprint. This is oscillation, and it will not resolve itself with more turns. Terminate and open a draft PR showing the wall it hit.

5. 5
   Stop 5 · Budget exhausted$0.77 modelled

   Twelve turns or $2.50, whichever binds first. The dollar check runs before the turn, not after, so the ceiling is a ceiling.

6. 6
   Stop 6 · Merge gate blocks$0.61 + human triage

   CI red, diff outside declared bounds, a forbidden path touched, or the critic flags a mismatch between the change and the requirement. Draft PR with the failing check named.

import { z } from "zod";
import micromatch from "micromatch";

export const CriticSchema = z.object({
  accomplishes_task: z.boolean(),
  does_extra_work: z.boolean(),
  concerns: z.array(z.string()).max(5),
});

const FORBIDDEN = [
  "**/migrations/**",
  "**/auth/**",
  "**/billing/**",
  "**/payments/**",
  ".github/workflows/**",
  "infra/**",
  "**/*.tf",
  "package-lock.json",
  "pnpm-lock.yaml",
];

const DIFF_LINE_MULTIPLIER = 3;

type Candidate = {
  task: string;
  diff: string;
  filesTouched: string[];
  filesDeclared: string[];
  linesChanged: number;
  linesPlanned: number;
  ciGreen: boolean;
  testsFlipped: string[];
};

export type GateResult =
  | { decision: "ready_for_review"; notes: string[] }
  | { decision: "blocked"; reason: string; detail?: string };

export async function mergeGate(
  c: Candidate,
  critic: (c: Candidate) => Promise<z.infer<typeof CriticSchema>>,
): Promise<GateResult> {
  // 1. free: the real pipeline, not the sandbox subset
  if (!c.ciGreen) {
    return { decision: "blocked", reason: "ci_red" };
  }

  // 2. free: forbidden paths are a hard stop, never a warning
  const forbidden = micromatch(c.filesTouched, FORBIDDEN);
  if (forbidden.length > 0) {
    return { decision: "blocked", reason: "forbidden_path", detail: forbidden.join(", ") };
  }

  // 3. free: the plan is a contract, not a suggestion
  const undeclared = c.filesTouched.filter((f) => !c.filesDeclared.includes(f));
  if (undeclared.length > 0) {
    return { decision: "blocked", reason: "undeclared_files", detail: undeclared.join(", ") };
  }

  // 4. free: a green diff 9x the planned size is still a red flag
  const ceiling = Math.max(40, c.linesPlanned * DIFF_LINE_MULTIPLIER);
  if (c.linesChanged > ceiling) {
    return {
      decision: "blocked",
      reason: "diff_too_large",
      detail: c.linesChanged + " changed vs ceiling " + ceiling,
    };
  }

  // 5. the only model call, on a cheap model, without the agent's reasoning
  const verdict = await critic(c);
  if (!verdict.accomplishes_task) {
    return { decision: "blocked", reason: "critic_task_mismatch", detail: verdict.concerns.join("; ") };
  }

  const notes = verdict.does_extra_work
    ? ["Critic flagged work beyond the task: " + verdict.concerns.join("; ")]
    : [];

  // The best outcome this function can produce is a PR a human still has to merge.
  return { decision: "ready_for_review", notes };
}

import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import { z } from "zod";

const server = new McpServer({ name: "sandbox-exec", version: "1.4.0" });

const MAX_FAILURES = 12;
const MAX_CHARS = 8000;

server.registerTool(
  "run_tests",
  {
    title: "Run a named subset of the test suite in this run's sandbox",
    description:
      "Runs the given test files or test names inside the sandbox for the " +
      "current run and returns pass/fail plus the first failures with stack " +
      "traces. Call this after every patch. Do NOT call this to explore the " +
      "codebase - use search_code or read_file, which are free and instant. " +
      "Do NOT pass an empty selector to run the whole suite; the full suite " +
      "runs once in CI after the branch is pushed.",
    inputSchema: {
      selectors: z.array(z.string()).min(1).max(40)
        .describe("Test file paths or fully qualified test names"),
      timeoutSec: z.number().int().min(5).max(300).default(120),
    },
  },
  async ({ selectors, timeoutSec }, extra) => {
    // Sandbox is NOT a model-settable argument. It is resolved from the
    // verified run context. This is the entire run-isolation story.
    const runId = requireRunId(extra.requestInfo);
    const sandbox = await sandboxes.forRun(runId);

    const started = Date.now();
    const exec = await sandbox.exec({
      cmd: testCommandFor(sandbox.language, selectors),
      timeoutSec,
      network: "none", // no egress, ever, including from the test process
    });

    const parsed = parseTestOutput(exec.stdout, exec.stderr);
    const failures = parsed.failures.slice(0, MAX_FAILURES).map((f) => ({
      test: f.name,
      message: f.message,
      trace: f.trace.split("\n").slice(0, 8).join("\n"),
    }));

    const payload = {
      passed: parsed.failed === 0 && exec.exitCode === 0,
      total: parsed.total,
      failed: parsed.failed,
      truncatedFailures: Math.max(0, parsed.failures.length - MAX_FAILURES),
      durationMs: Date.now() - started,
      timedOut: exec.timedOut,
      failures,
    };

    const text = JSON.stringify(payload).slice(0, MAX_CHARS);

    return {
      // A failing suite is the signal the agent needs, not a tool error.
      content: [{ type: "text", text }],
      structuredContent: payload,
    };
  },
);

1. Week 1
   Intake, journal, cost attribution

   Task schema with acceptance criteria and file allowlist. Run and turn tables. Cost middleware storing micros against a rate-card id. Zero demo value, total downstream dependency.

2. Week 2
   The sandbox

   Disposable microVM that installs dependencies and runs your suite with egress denied. Per-second billing, hard TTL, clean teardown. Platform work that decides whether this passes a security review.

3. Weeks 3‑4
   Repo grounding

   Symbol graph via tree-sitter or the language server, incremental per-commit refresh, hybrid chunk index, past-PR corpus with review comments. The parse is a day; freshness is the rest.

4. Week 5
   Planner and edit loop

   Typed plan, declared file set enforced by the loop, turn and dollar budgets, no-progress fingerprinting, journal writes per turn. Smaller than anyone expects.

5. Week 6
   MCP tools and per-run credentials

   Ten tools, branch-scoped token minted per run with a one-hour TTL, no force-push, no workflow writes, no merge. More security design than code.

6. Weeks 7‑8
   Merge gate and critic calibration

   Deterministic checks are quick. Calibrating the critic against a real set of merged and rejected diffs is the work, and skipping it means your gate is a guess wearing a schema.

7. Weeks 9‑10
   Evals, evidence, rollout

   Deterministic CI suite over a frozen task set, dashboards, the PR-body template, review-queue rate limiting, and shadow mode on a real repo where nobody merges anything for two weeks.