LLM Routing and Caching: How to Actually Price a Request

create table price_snapshot (
  id                        uuid primary key,
  provider                  text not null,
  model                     text not null,
  model_version             text not null,     -- pin it. -latest is not a version.
  input_micros_per_mtok         bigint not null,
  cached_input_micros_per_mtok  bigint not null,
  cache_write_5m_micros_per_mtok bigint,
  cache_write_1h_micros_per_mtok bigint,
  output_micros_per_mtok        bigint not null,
  effective_from            timestamptz not null,
  effective_to              timestamptz,        -- null = current
  source_url                text not null,      -- the vendor pricing page
  captured_at               timestamptz not null default now(),
  captured_by               text not null,      -- 'scraper' | 'neeraj' | ...
  unique (provider, model, model_version, effective_from)
);

create unique index one_current_card
  on price_snapshot (provider, model, model_version)
  where effective_to is null;

create table llm_call (
  id                 uuid primary key,
  trace_id           text not null,
  run_id             uuid,
  tenant_id          uuid not null,
  feature            text not null,      -- what the user was doing
  request_class      text not null,      -- the routing + caching join key
  provider           text not null,
  model              text not null,
  model_version      text not null,
  input_tokens       int  not null,
  cached_input_tokens int not null default 0,
  cache_write_tokens int  not null default 0,
  output_tokens      int  not null,
  price_snapshot_id  uuid not null references price_snapshot(id),
  cost_micros        bigint not null,    -- computed ONCE, at call time
  latency_ms         int  not null,
  created_at         timestamptz not null default now()
);

create index on llm_call (tenant_id, created_at desc);
create index on llm_call (request_class, created_at desc);
create index on llm_call (price_snapshot_id);

-- The counterfactual that is impossible without snapshots:
-- what would last month have cost on the CURRENT rate card?
select
  c.request_class,
  sum(c.cost_micros) / 1e6                          as actual_usd,
  sum( c.input_tokens        * n.input_micros_per_mtok
     + c.cached_input_tokens * n.cached_input_micros_per_mtok
     + c.output_tokens       * n.output_micros_per_mtok ) / 1e12 as at_new_prices_usd
from llm_call c
join price_snapshot n
  on  n.provider = c.provider
  and n.model    = c.model
  and n.model_version = c.model_version
  and n.effective_to is null
where c.created_at >= date_trunc('month', now()) - interval '1 month'
  and c.created_at <  date_trunc('month', now())
group by 1
order by actual_usd desc;

type Block = { role: "system" | "user" | "assistant"; text: string };

export function assemble(req: Request, tools: ToolDef[], hist: Turn[]): Block[] {
  const blocks: Block[] = [];

  // ---- STABLE ZONE: byte-identical across every request of this class ----
  // KILLER 1: never interpolate a timestamp, request id, or tenant name here.
  blocks.push({ role: "system", text: SYSTEM_PROMPT_V7 });

  // KILLER 2: sort tools by name and serialise deterministically.
  // A Map iterated in insertion order gives different bytes per process.
  const catalogue = [...tools]
    .sort((a, b) => a.name.localeCompare(b.name))
    .map(stableStringify)
    .join("\n");
  blocks.push({ role: "system", text: catalogue });

  blocks.push({ role: "system", text: FEW_SHOT_V3 });
  // <-- Anthropic cache breakpoint goes here. Everything above is the prefix.

  // ---- SEMI-STABLE ZONE ----
  // KILLER 3: personalisation above this line multiplies cache entries by
  // your tenant count. 500 tenants x 288 writes/day x 9,000 tokens at
  // 1.25x mid-tier input is roughly $97,200/month of pure write cost.
  if (req.tenantPreamble) blocks.push({ role: "system", text: req.tenantPreamble });

  // ---- VOLATILE ZONE: different every request, never cacheable ----
  for (const t of hist) blocks.push({ role: t.role, text: t.text });
  blocks.push({ role: "user", text: req.userText });

  return blocks;
}

/** The only reliable detector. Without it, a zeroed hit rate is found
 *  on next month's invoice rather than in this afternoon's deploy. */
export function assertCacheHealth(m: CallMetrics, expectPrefix: number) {
  const ratio = m.cachedInputTokens / Math.max(expectPrefix, 1);
  if (ratio < 0.5) {
    metrics.increment("llm.cache.prefix_miss", { class: m.requestClass });
    log.warn("prefix cache miss", {
      class: m.requestClass, cached: m.cachedInputTokens,
      expected: expectPrefix, model_version: m.modelVersion,
    });
  }
}

$ $ psql -c "select tenant_id, sum(cost_micros)/1e6 usd, count(*) calls
$          from llm_call where created_at > now() - interval '30 days'
$          group by 1 order by usd desc limit 3"
 tenant_id                             |    usd   | calls
 4c1e...9a  (Northwind, $499/mo plan)  | 1,284.40 | 61,203   <-- unprofitable
 77bd...02  (Contoso,  $2,400/mo plan) |   611.02 | 39,880
 0af3...d1  (Fabrikam, $999/mo plan)   |   402.77 | 24,101
$ $ costctl class-trend --weeks 2 --alert-at 25%
 request_class   last_wk/req   this_wk/req   delta
 rag_answer       $0.01168      $0.01191      +2.0%   ok
 agent_step       $0.03630      $0.05942     +63.7%   ALERT
 classify         $0.00016      $0.00016      +0.0%   ok
$ $ costctl explain agent_step --since 2026-08-17
 cached_input_tokens p50: 9,014 -> 0        <-- prefix cache died
 deploy 4f2a1c 'add build sha to system prompt' at 2026-08-18T09:12Z
 modelled monthly impact: +$2,312 on this class alone
$ $ costctl retries --class agent_step --month 2026-08
 attempts>1: 5.1% of 100,412 steps, 10,240 extra calls, $525.32
 (invisible in any dashboard built from success-path logs only)