Managing AI Technical Debt

ai_technical_debt:
  prompt_debt:
    description: "Prompts that work but are fragile"
    symptoms:
      - Prompt depends on model quirks
      - No documentation of why it works
      - Breaks on model updates
    cost: "Maintenance burden, brittleness"

  evaluation_debt:
    description: "Missing or inadequate evaluation"
    symptoms:
      - No automated quality checks
      - Manual testing only
      - Unknown failure modes
    cost: "Quality issues, blind spots"

  data_debt:
    description: "Data quality and pipeline issues"
    symptoms:
      - Stale embeddings
      - Missing documents
      - No freshness monitoring
    cost: "Degraded results, inconsistency"

  architecture_debt:
    description: "Expedient but problematic design"
    symptoms:
      - Hard-coded model references
      - No abstraction layers
      - Tight coupling
    cost: "Difficult changes, vendor lock-in"

debt_indicators:
  code_level:
    - Magic numbers in prompts
    - Copy-pasted prompt variations
    - No type hints on AI interfaces
    - Catch-all exception handlers

  system_level:
    - No evaluation suite
    - Unknown cost per feature
    - Manual deployment process
    - No observability

  operational:
    - Fear of model updates
    - "Don't touch" prompt files
    - Undocumented workarounds
    - Quality complaints increasing

# Before: Fragile prompt
PROMPT = """You are an assistant. Be helpful.
When the user asks about products, give good info.
Don't make stuff up. Be nice."""

# After: Structured, documented prompt
@dataclass
class AssistantPrompt:
    """
    Product assistant prompt.

    Rationale:
    - Role statement establishes context
    - Explicit constraints prevent hallucination
    - Format guidance ensures consistency

    Tested on: GPT-4o, Claude 3.5 Sonnet
    Last updated: 2025-10-15
    """

    role: str = "You are a product information assistant."

    constraints: list[str] = field(default_factory=lambda: [
        "Only provide information from the product database",
        "If information is not available, say so clearly",
        "Never make up product features or prices"
    ])

    format: str = "Respond concisely with relevant product details."

    def render(self) -> str:
        return f"""{self.role}

Constraints:
{chr(10).join(f'- {c}' for c in self.constraints)}

{self.format}"""

class EvaluationDebtPayoff:
    """Build evaluation suite to pay down debt."""

    async def create_evaluation_suite(
        self,
        feature: str
    ) -> EvaluationSuite:
        # Collect production examples
        samples = await self.logs.sample_requests(
            feature=feature,
            count=100
        )

        # Generate test cases
        test_cases = []
        for sample in samples:
            test_cases.append(TestCase(
                input=sample.input,
                expected_properties=await self._infer_properties(sample),
                golden_response=sample.response if sample.was_good else None
            ))

        # Create automated checks
        checks = [
            FormatCheck(feature=feature),
            SafetyCheck(),
            RelevanceCheck(),
            FactualityCheck(knowledge_base=self.kb)
        ]

        return EvaluationSuite(
            name=f"{feature}_eval",
            test_cases=test_cases,
            checks=checks
        )

# Before: Tight coupling
class ChatService:
    def __init__(self):
        self.client = openai.OpenAI()

    def chat(self, message: str) -> str:
        return self.client.chat.completions.create(
            model="gpt-4o",
            messages=[{"role": "user", "content": message}]
        ).choices[0].message.content

# After: Abstracted, testable
class ChatService:
    def __init__(self, llm: LLMProvider):
        self.llm = llm

    async def chat(self, message: str) -> str:
        return await self.llm.generate(
            messages=[{"role": "user", "content": message}]
        )

# LLM provider abstraction
class LLMProvider(Protocol):
    async def generate(self, messages: list[dict]) -> str: ...

class OpenAIProvider(LLMProvider):
    async def generate(self, messages: list[dict]) -> str: ...

class AnthropicProvider(LLMProvider):
    async def generate(self, messages: list[dict]) -> str: ...

class MockProvider(LLMProvider):
    async def generate(self, messages: list[dict]) -> str:
        return "Mock response for testing"

ai_debt_prevention:
  coding_standards:
    - Prompts in version control
    - Documentation requirements
    - Abstraction layers
    - Type hints everywhere

  process:
    - Evaluation before deployment
    - Model update testing
    - Regular debt review
    - Refactoring time allocated

  monitoring:
    - Quality metrics tracked
    - Cost visibility
    - Drift detection
    - Freshness alerts