AI Product Metrics That Matter

metrics_comparison:
  traditional:
    - DAU/MAU
    - Conversion rate
    - Time in app
    - Feature adoption
    - Tells you: "Are users using it?"

  ai_specific:
    - Output quality
    - Task success rate
    - Hallucination rate
    - User trust
    - Tells you: "Is it actually good?"

  the_gap:
    - Users might use bad AI
    - High engagement ≠ high quality
    - Need both types of metrics

class QualityMetrics:
    """Track AI output quality."""

    async def measure_quality(
        self,
        request: Request,
        response: Response,
        context: dict
    ) -> QualityScore:
        scores = {}

        # Automated evaluation
        scores["automated"] = await self._automated_eval(
            request, response
        )

        # User feedback (when available)
        if context.get("user_feedback"):
            scores["user"] = context["user_feedback"]

        # Human evaluation (sampled)
        if self._should_sample():
            scores["human"] = await self._queue_for_human_eval(
                request, response
            )

        return QualityScore(
            overall=self._aggregate(scores),
            breakdown=scores
        )

    async def _automated_eval(
        self,
        request: Request,
        response: Response
    ) -> dict:
        checks = await asyncio.gather(
            self._check_relevance(request, response),
            self._check_coherence(response),
            self._check_safety(response),
            self._check_format(response)
        )

        return {
            "relevance": checks[0],
            "coherence": checks[1],
            "safety": checks[2],
            "format": checks[3]
        }

class TaskSuccessTracker:
    """Track whether AI helps users complete tasks."""

    async def track_task(
        self,
        task_id: str,
        ai_interactions: list[Interaction],
        outcome: TaskOutcome
    ):
        await self.store.record(
            task_id=task_id,
            interactions=len(ai_interactions),
            outcome=outcome.status,
            time_to_completion=outcome.duration,
            ai_contribution=self._assess_contribution(
                ai_interactions, outcome
            )
        )

    def get_success_rate(
        self,
        time_period: str = "7d"
    ) -> SuccessMetrics:
        tasks = self.store.query(period=time_period)

        return SuccessMetrics(
            completion_rate=self._completion_rate(tasks),
            ai_assisted_rate=self._ai_assisted_rate(tasks),
            ai_success_rate=self._ai_success_rate(tasks),
            avg_interactions=self._avg_interactions(tasks)
        )

trust_metrics:
  explicit:
    - Thumbs up/down on responses
    - "Was this helpful?" surveys
    - Reported issues

  implicit:
    - Response acceptance rate
    - Edit rate after AI suggestion
    - Return usage patterns
    - Abandonment after AI response

  longitudinal:
    - Trust over time
    - Usage patterns changes
    - Feature dependency

ai_product_scorecard:
  quality:
    metrics:
      - Accuracy/correctness
      - Relevance
      - Safety
    targets:
      - ">90% accuracy on evals"
      - "<1% safety violations"

  utility:
    metrics:
      - Task completion rate
      - Time saved
      - User satisfaction
    targets:
      - ">70% task completion"
      - "30% time reduction"

  reliability:
    metrics:
      - Availability
      - Latency
      - Consistency
    targets:
      - "99.9% uptime"
      - "<3s P95 latency"

  efficiency:
    metrics:
      - Cost per interaction
      - Cost per successful task
    targets:
      - "Declining cost trend"

class AIProductDashboard:
    """Comprehensive AI product metrics."""

    def get_dashboard_data(self, period: str) -> DashboardData:
        return DashboardData(
            # Usage (traditional)
            usage=UsageMetrics(
                active_users=self.get_active_users(period),
                interactions=self.get_interaction_count(period),
                feature_adoption=self.get_adoption_rate(period)
            ),

            # Quality (AI-specific)
            quality=QualityMetrics(
                accuracy=self.get_accuracy(period),
                safety_rate=self.get_safety_rate(period),
                hallucination_rate=self.get_hallucination_rate(period)
            ),

            # Success (AI-specific)
            success=SuccessMetrics(
                task_completion=self.get_task_completion(period),
                user_satisfaction=self.get_satisfaction(period),
                trust_score=self.get_trust_score(period)
            ),

            # Efficiency
            efficiency=EfficiencyMetrics(
                cost_per_interaction=self.get_cost_per_interaction(period),
                latency_p50=self.get_latency_p50(period),
                latency_p95=self.get_latency_p95(period)
            )
        )