AI Performance Metrics Mastery

# Simple AI Metrics Tracker class AIMetricsTracker: def __init__(self): self.predictions = [] self.actual_values = [] self.user_feedback = [] self.response_times = [] # Track a prediction def log_prediction(self, predicted, actual, user_rating, response_time): self.predictions.append(predicted) self.actual_values.append(actual) self.user_feedback.append(user_rating) self.response_times.append(response_time) # Calculate accuracy def get_accuracy(self): if not self.predictions: return 0 correct = sum(1 for p, a in zip(self.predictions, self.actual_values) if p == a) return correct / len(self.predictions) * 100 # Get user satisfaction def get_satisfaction(self): if not self.user_feedback: return 0 return sum(self.user_feedback) / len(self.user_feedback) # Get average response time def get_avg_response_time(self): if not self.response_times: return 0 return sum(self.response_times) / len(self.response_times) # Example usage tracker = AIMetricsTracker() tracker.log_prediction("spam", "spam", 5, 0.1) # Correct prediction, 5-star rating, 0.1s response tracker.log_prediction("ham", "spam", 2, 0.2) # Wrong prediction, 2-star rating print(f"Accuracy: {tracker.get_accuracy():.1f}%") # 50.0% print(f"Satisfaction: {tracker.get_satisfaction():.1f}/5") # 3.5/5 print(f"Avg Response: {tracker.get_avg_response_time():.2f}s") # 0.15s

class HEARTMetrics: def __init__(self): self.user_ratings = [] self.sessions = {} self.new_users = set() self.returning_users = set() self.task_completions = [] # Happiness: Track user satisfaction def record_happiness(self, user_id, rating, feedback=None): self.user_ratings.append({ 'user_id': user_id, 'rating': rating, 'feedback': feedback, 'timestamp': datetime.now() }) # Engagement: Track session activity def record_session(self, user_id, duration_minutes, actions_taken): if user_id not in self.sessions: self.sessions[user_id] = [] self.sessions[user_id].append({ 'duration': duration_minutes, 'actions': actions_taken, 'date': datetime.now().date() }) # Task Success: Did user complete their goal? def record_task_attempt(self, user_id, task_type, completed, time_to_complete=None): self.task_completions.append({ 'user_id': user_id, 'task': task_type, 'completed': completed, 'time_to_complete': time_to_complete }) # Calculate key metrics def get_happiness_score(self): if not self.user_ratings: return 0 total_rating = sum(r['rating'] for r in self.user_ratings) return total_rating / len(self.user_ratings) def get_task_success_rate(self): if not self.task_completions: return 0 completed = sum(1 for t in self.task_completions if t['completed']) return completed / len(self.task_completions) * 100 # Example usage heart = HEARTMetrics() heart.record_happiness("user123", 4, "Great AI recommendations!") heart.record_task_attempt("user123", "find_product", True, 2.5) print(f"Happiness Score: {heart.get_happiness_score()}/5") print(f"Task Success Rate: {heart.get_task_success_rate():.1f}%")

import random from scipy import stats class ABTestFramework: def __init__(self, test_name): self.test_name = test_name self.group_a_results = [] # Control group self.group_b_results = [] # Test group self.user_assignments = {} # Randomly assign users to groups def assign_user_to_group(self, user_id, split_ratio=0.5): if user_id not in self.user_assignments: # Assign to A or B group self.user_assignments[user_id] = 'A' if random.random() < split_ratio else 'B' return self.user_assignments[user_id] # Record results for each group def record_result(self, user_id, success, metric_value): group = self.assign_user_to_group(user_id) result = { 'user_id': user_id, 'success': success, 'metric_value': metric_value } if group == 'A': self.group_a_results.append(result) else: self.group_b_results.append(result) # Calculate statistical significance def get_results(self): if len(self.group_a_results) < 30 or len(self.group_b_results) < 30: return "Need more data for statistical significance" # Success rates success_a = sum(1 for r in self.group_a_results if r['success']) / len(self.group_a_results) success_b = sum(1 for r in self.group_b_results if r['success']) / len(self.group_b_results) # Statistical test a_successes = sum(1 for r in self.group_a_results if r['success']) b_successes = sum(1 for r in self.group_b_results if r['success']) # Chi-square test for significance chi2, p_value = stats.chi2_contingency([ [a_successes, len(self.group_a_results) - a_successes], [b_successes, len(self.group_b_results) - b_successes] ])[:2] return { 'group_a_success_rate': success_a * 100, 'group_b_success_rate': success_b * 100, 'improvement': ((success_b - success_a) / success_a) * 100, 'p_value': p_value, 'significant': p_value < 0.05 }

import pandas as pd from datetime import datetime, timedelta class CohortAnalyzer: def __init__(self): self.user_data = [] # Track user interactions with AI def log_interaction(self, user_id, interaction_date, ai_feature_used, success): self.user_data.append({ 'user_id': user_id, 'date': interaction_date, 'feature': ai_feature_used, 'success': success }) # Analyze user retention by signup cohort def analyze_retention_by_cohort(self): df = pd.DataFrame(self.user_data) # Get first interaction date for each user (signup) user_cohorts = df.groupby('user_id')['date'].min().reset_index() user_cohorts['cohort_month'] = user_cohorts['date'].dt.to_period('M') # Add cohort info back to main data df = df.merge(user_cohorts[['user_id', 'cohort_month']], on='user_id') # Calculate months since signup df['period_month'] = df['date'].dt.to_period('M') df['months_since_signup'] = (df['period_month'] - df['cohort_month']).apply(attrgetter('n')) # Calculate retention rates cohort_data = df.groupby(['cohort_month', 'months_since_signup'])['user_id'].nunique().reset_index() cohort_sizes = df.groupby('cohort_month')['user_id'].nunique() cohort_table = cohort_data.pivot(index='cohort_month', columns='months_since_signup', values='user_id') # Convert to percentages cohort_percentages = cohort_table.divide(cohort_sizes, axis=0) * 100 return cohort_percentages # Analyze AI feature adoption over time def analyze_feature_adoption(self): df = pd.DataFrame(self.user_data) feature_adoption = df.groupby(['date'.dt.date, 'feature']).agg({ 'user_id': 'nunique', # Unique users 'success': ['count', 'sum'] # Total attempts, successes }).round(2) return feature_adoption # Example: Track AI chatbot usage analyzer = CohortAnalyzer() # Simulate user data for month in range(6): # 6 months of data base_date = datetime(2024, 1, 1) + timedelta(days=30*month) for user in range(100): # 100 users per month analyzer.log_interaction( user_id=f"user_{month}_{user}", interaction_date=base_date + timedelta(days=random.randint(0, 29)), ai_feature_used=random.choice(["chatbot", "recommendations", "search"]), success=random.random() > 0.2 # 80% success rate ) retention_analysis = analyzer.analyze_retention_by_cohort() print("Cohort Retention Analysis:") print(retention_analysis)