Why this matters

Context windows have grown dramatically — 128K, 200K, even 1M tokens are available. It is tempting to assume the budget problem is solved. It is not.

Large context windows have costs: latency increases with input length, pricing scales with tokens, and model attention quality degrades with excessive noise. The skill of context budgeting — allocating a finite token budget across competing sources — remains one of the highest-leverage practices in context engineering.

Core concepts

Token counting

import tiktoken


def count_tokens(text: str, model: str = "cl100k_base") -> int:
    enc = tiktoken.get_encoding(model)
    return len(enc.encode(text))


def count_messages_tokens(messages: list[dict], model: str = "cl100k_base") -> int:
    enc = tiktoken.get_encoding(model)
    total = 0
    for message in messages:
        total += 4  # overhead per message
        total += len(enc.encode(message.get("content", "")))
    total += 2  # assistant turn priming
    return total


def estimate_tokens(text: str) -> int:
    # ~4 chars per token, accurate to within 15% for English prose
    return max(4, int(len(text) / 4))

Priority-based allocation

from dataclasses import dataclass


@dataclass
class BudgetAllocation:
    system_prompt: int
    conversation_history: int
    retrieved_context: int
    tool_results: int
    user_input: int

    @property
    def total(self) -> int:
        return (
            self.system_prompt + self.conversation_history
            + self.retrieved_context + self.tool_results + self.user_input
        )


# For a 16K context window, leaving 2K for the response:
allocation = BudgetAllocation(
    system_prompt=1500,
    conversation_history=4000,
    retrieved_context=6000,
    tool_results=2000,
    user_input=500,
)

Truncation strategies

Sliding window — keep the most recent N tokens:

def sliding_window(messages: list[dict], max_tokens: int, count_fn=estimate_tokens) -> list[dict]:
    result, total = [], 0
    for msg in reversed(messages):
        cost = count_fn(msg["content"])
        if total + cost > max_tokens:
            break
        result.append(msg)
        total += cost
    return list(reversed(result))

Importance scoring — keep the highest-scoring chunks:

@dataclass
class ScoredChunk:
    content: str
    score: float


def importance_truncate(
    chunks: list[ScoredChunk], max_tokens: int, count_fn=estimate_tokens
) -> list[ScoredChunk]:
    sorted_chunks = sorted(chunks, key=lambda c: c.score, reverse=True)
    selected, total = [], 0
    for chunk in sorted_chunks:
        cost = count_fn(chunk.content)
        if total + cost <= max_tokens:
            selected.append(chunk)
            total += cost
    return sorted(selected, key=lambda c: chunks.index(c))

Summarization — compress rather than drop:

async def summarize_history(messages: list[dict], client) -> str:
    transcript = "
".join(f"{m['role'].upper()}: {m['content']}" for m in messages)
    response = await client.messages.create(
        model="claude-3-haiku-20240307",
        max_tokens=300,
        messages=[{"role": "user", "content": (
            "Summarize the following conversation in 3-5 sentences. "
            "Focus on decisions made and the current state of tasks.

" + transcript
        )}],
    )
    return response.content[0].text

Code: ContextBudgetManager

from dataclasses import dataclass
from typing import Callable


@dataclass
class ContextBudgetManager:
    total_context_window: int
    response_reserve: int = 2048
    count_fn: Callable[[str], int] = estimate_tokens

    @property
    def available_tokens(self) -> int:
        return self.total_context_window - self.response_reserve

    def allocate(
        self,
        system_prompt: str,
        history: list[dict],
        retrieved_docs: list[str],
        tool_results: list[str],
        user_input: str,
    ) -> list[dict]:
        budget = self.available_tokens
        budget -= self.count_fn(system_prompt)
        budget -= self.count_fn(user_input)

        tool_budget = int(budget * 0.25)
        tool_content = self._fit_items(tool_results, tool_budget)

        retrieval_budget = int(budget * 0.50)
        doc_content = self._fit_items(retrieved_docs, retrieval_budget)

        history_budget = budget - tool_budget - retrieval_budget
        trimmed_history = sliding_window(history, history_budget, self.count_fn)

        messages: list[dict] = [{"role": "system", "content": system_prompt}]
        if doc_content:
            messages.append({"role": "user", "content": "Relevant context:
" + doc_content})
            messages.append({"role": "assistant", "content": "I have reviewed the context."})
        messages.extend(trimmed_history)
        if tool_content:
            messages.append({"role": "user", "content": "Tool results:
" + tool_content})
            messages.append({"role": "assistant", "content": "I have reviewed the tool results."})
        messages.append({"role": "user", "content": user_input})
        return messages

    def _fit_items(self, items: list[str], budget: int) -> str:
        selected, total = [], 0
        for item in items:
            cost = self.count_fn(item)
            if total + cost <= budget:
                selected.append(item)
                total += cost
        return "

---

".join(selected)

Common mistakes

Forgetting the response reserve. If your context window is 8K and you fill 8K with input, the model has zero tokens to respond.

Token counting the wrong model. Add a 10-15% safety margin when approximating across different tokenizers.

Static allocations that never adapt. Dynamic allocation makes better use of available budget.

Counting tokens on every call. Cache token counts for content that rarely changes.

Try it yourself

1. Measure the actual token counts for the system prompt, conversation history, and retrieved documents in one of your LLM applications.
2. Implement a sliding_window that preserves the first message in history even when it would normally be dropped.
3. Compare estimate_tokens against tiktoken for five different text samples.