RAG Over Private Documents: The Full Architecture, Failure Modes, and Cost Per Answer

1. 1
   1. Is the fact present in any raw chunk?2 minutes · SQL

   Full-text search your chunk table for a distinctive phrase from the correct answer. No hit means the failure is upstream of retrieval entirely — a parser or a chunker dropped it, and no amount of prompt work will recover it.

2. 2
   2. Is that chunk in the top 50 of raw retrieval?3 minutes · one script

   Run retrieval with the reranker and the ACL filter disabled. If the chunk ranks below 50, retrieval is your problem: usually dense-only search on a query full of identifiers, which BM25 would have caught.

3. 3
   3. Is it in the final context after rerank and ACL?2 minutes · trace

   Log the chunk ids that actually entered the prompt. If the right chunk was in the top 50 and is not in the context, it was dropped by the reranker or the permission filter — and the permission filter is the more likely of the two.

4. 4
   4. Does the model answer correctly with only that chunk?3 minutes · one call

   Hand-build a context containing exactly the right chunk and re-ask. A wrong answer here is a generation problem. A correct answer here means the problem is distractors or ordering in context assembly, which is a different fix.

-- indexes that make this work
create index on chunk using gin  (acl_principals);
create index on chunk using gin  (tsv);
create index on chunk using hnsw (embedding vector_cosine_ops)
  with (m = 16, ef_construction = 64);
create index on chunk (tenant_id, embedding_model_version);

-- :over_fetch — derive from measured selectivity, do not guess.
-- If the median caller can see 8% of the corpus, 50 final candidates
-- means asking the index for ~200 before fusion.

with permitted as (
  select chunk_id, text, heading_path, embedding, tsv
  from chunk
  where tenant_id = :tenant
    and embedding_model_version = :emb_version   -- never mix versions
    and acl_principals && :principals            -- THE pre-filter
    and (effective_to is null or effective_to > now())
),
dense as (
  select chunk_id,
         row_number() over (order by embedding <=> :qvec) as rank
  from permitted
  order by embedding <=> :qvec
  limit :over_fetch
),
lexical as (
  select chunk_id,
         row_number() over (order by ts_rank_cd(tsv, plainto_tsquery(:q)) desc) as rank
  from permitted
  where tsv @@ plainto_tsquery(:q)
  limit :over_fetch
)
select c.chunk_id, c.heading_path, c.text,
       coalesce(1.0 / (60 + d.rank), 0) +
       coalesce(1.0 / (60 + l.rank), 0) as rrf_score
from permitted c
left join dense   d on d.chunk_id = c.chunk_id
left join lexical l on l.chunk_id = c.chunk_id
where d.chunk_id is not null or l.chunk_id is not null
order by rrf_score desc
limit 50;

create table chunk (
  chunk_id        uuid primary key,
  doc_id          uuid not null,
  doc_version     int  not null,
  tenant_id       uuid not null,
  source_uri      text not null,
  heading_path    text[] not null default '{}',   -- ['Policies','Leave','Carry-over']
  parent_chunk_id uuid,                            -- retrieve small, generate large
  text            text not null,
  token_count     int  not null,
  effective_from  date,
  effective_to    date,                            -- null = current
  acl_principals  text[] not null,                 -- GIN indexed; the pre-filter
  embedding       vector(1536) not null,
  embedding_model_version text not null,           -- never mix versions in one index
  tsv             tsvector generated always as
                    (to_tsvector('english', text)) stored,
  ingested_at     timestamptz not null default now()
);

-- the tombstone path: three deletes, one reconciler.
-- deleting a document must remove it from all three surfaces.
create or replace function tombstone_doc(p_doc uuid) returns void as $$
begin
  delete from chunk        where doc_id = p_doc;   -- vector + lexical, same row
  delete from answer_cache where p_doc = any(source_doc_ids);
  insert into tombstone (doc_id, removed_at) values (p_doc, now());
end; $$ language plpgsql;

const AnswerSchema = {
  type: "object",
  required: ["answer", "citations", "sufficient"],
  properties: {
    sufficient: { type: "boolean" },
    answer:     { type: "string" },
    citations:  {
      type: "array",
      items: {
        type: "object",
        required: ["chunk_id", "quote"],
        properties: {
          chunk_id: { type: "string" },
          quote:    { type: "string", minLength: 12 },
        },
      },
    },
  },
} as const;

const norm = (s: string) =>
  s.replace(/[\u2018\u2019]/g, "'")
   .replace(/[\u201C\u201D]/g, '"')
   .replace(/\s+/g, " ")
   .trim()
   .toLowerCase();

export function validateCitations(
  out: Answer, chunks: Map<string, string>,
): { ok: true } | { ok: false; bad: Citation[] } {
  const bad = out.citations.filter(c => {
    const src = chunks.get(c.chunk_id);
    return !src || !norm(src).includes(norm(c.quote));
  });
  return bad.length === 0 ? { ok: true } : { ok: false, bad };
}

// one mutated retry, then refuse. Never an identical retry.
export async function answerWithCitations(ctx: Ctx): Promise<Answer> {
  let out = await ctx.model.json(AnswerSchema, ctx.messages);
  const v = validateCitations(out, ctx.chunks);
  if (v.ok) return out;

  ctx.messages.push(userMessage(
    "These citations do not appear in the chunks you attributed them to: " +
    JSON.stringify(v.bad) +
    ". Quote exactly, or set sufficient=false and explain what is missing."
  ));
  out = await ctx.model.json(AnswerSchema, ctx.messages);
  return validateCitations(out, ctx.chunks).ok
    ? out
    : { sufficient: false, answer: REFUSAL, citations: [] };
}

"""Retrieval-only eval. No judge, no generation. Python 3.10+."""
from dataclasses import dataclass
from statistics import mean

@dataclass
class Case:
    question: str
    must_retrieve: set[str]     # human-confirmed chunk ids
    principals: list[str]       # evaluate WITH the caller's ACL, always

def recall_at_k(retrieved: list[str], gold: set[str], k: int) -> float:
    if not gold:
        return 1.0
    return len(gold & set(retrieved[:k])) / len(gold)

def mrr(retrieved: list[str], gold: set[str]) -> float:
    for i, cid in enumerate(retrieved, start=1):
        if cid in gold:
            return 1.0 / i
    return 0.0

def run(cases: list[Case], retrieve) -> dict[str, float]:
    r5, r20, r50, rr = [], [], [], []
    for c in cases:
        got = retrieve(c.question, principals=c.principals, k=50)
        ids = [x.chunk_id for x in got]
        r5.append(recall_at_k(ids, c.must_retrieve, 5))
        r20.append(recall_at_k(ids, c.must_retrieve, 20))
        r50.append(recall_at_k(ids, c.must_retrieve, 50))
        rr.append(mrr(ids, c.must_retrieve))
    return {
        "recall@5": mean(r5), "recall@20": mean(r20),
        "recall@50": mean(r50), "mrr": mean(rr), "n": len(cases),
    }

# Two rules that make this trustworthy:
#   1. Always evaluate with a real principal set. A retrieval eval run as
#      an admin measures a system no user has.
#   2. Stratify: identifiers, synonyms, negations, multi-hop, superseded
#      documents, and questions whose correct answer is a refusal.