# Import statistical and data libraries
import pandas as pd
import numpy as np
import scipy
from scipy import stats
from scipy.stats import ttest_ind, chi2_contingency, pearsonr, spearmanr
import warnings
warnings.filterwarnings('ignore')

print("✅ Libraries imported successfully!")
print(f"Pandas: {pd.__version__}")
print(f"NumPy: {np.__version__}")
print(f"SciPy: {scipy.__version__}")

# Load the survey response dataset
survey = pd.read_csv('/Users/bamr87/github/zer0-mistakes/assets/data/notebooks/survey_responses.csv')

print("📊 Survey Data Preview:")
print(f"Shape: {survey.shape[0]} respondents × {survey.shape[1]} questions\n")
survey.head(10)

# Calculate comprehensive descriptive statistics
numeric_cols = ['age', 'product_satisfaction', 'service_rating', 'feedback_length']

print("📈 Descriptive Statistics for Survey Responses:")
print("=" * 70)

for col in numeric_cols:
    data = survey[col]
    print(f"\n{col.upper().replace('_', ' ')}:")
    print(f"  Mean:     {data.mean():.2f}")
    print(f"  Median:   {data.median():.2f}")
    print(f"  Mode:     {data.mode().values[0]}")
    print(f"  Std Dev:  {data.std():.2f}")
    print(f"  Variance: {data.var():.2f}")
    print(f"  Range:    {data.min()} - {data.max()}")
    print(f"  IQR:      {data.quantile(0.75) - data.quantile(0.25):.2f}")

# Calculate correlation matrix for satisfaction metrics
satisfaction_cols = ['product_satisfaction', 'service_rating', 'feedback_length', 'age']
correlation_matrix = survey[satisfaction_cols].corr()

print("🔗 Correlation Matrix (Pearson):")
print(correlation_matrix.round(3))

# Detailed pairwise correlations with significance
print("\n\n📊 Detailed Correlation Analysis:")
print("=" * 60)

pairs = [
    ('product_satisfaction', 'service_rating'),
    ('age', 'product_satisfaction'),
    ('age', 'service_rating'),
    ('feedback_length', 'product_satisfaction')
]

for col1, col2 in pairs:
    r, p_value = pearsonr(survey[col1], survey[col2])
    significance = "***" if p_value < 0.001 else "**" if p_value < 0.01 else "*" if p_value < 0.05 else "ns"
    print(f"\n{col1} vs {col2}:")
    print(f"  Pearson r = {r:.4f} ({significance})")
    print(f"  p-value   = {p_value:.4f}")
    if abs(r) >= 0.7:
        strength = "strong"
    elif abs(r) >= 0.4:
        strength = "moderate"
    else:
        strength = "weak"
    direction = "positive" if r > 0 else "negative"
    print(f"  → {strength.capitalize()} {direction} correlation")

# Independent samples t-test: Compare satisfaction between genders
male_satisfaction = survey[survey['gender'] == 'Male']['product_satisfaction']
female_satisfaction = survey[survey['gender'] == 'Female']['product_satisfaction']

t_stat, p_value = ttest_ind(male_satisfaction, female_satisfaction)

print("🧪 Independent Samples T-Test: Product Satisfaction by Gender")
print("=" * 60)
print(f"\nGroup Statistics:")
print(f"  Male   (n={len(male_satisfaction)}):   M = {male_satisfaction.mean():.2f}, SD = {male_satisfaction.std():.2f}")
print(f"  Female (n={len(female_satisfaction)}): M = {female_satisfaction.mean():.2f}, SD = {female_satisfaction.std():.2f}")
print(f"\nTest Results:")
print(f"  t-statistic = {t_stat:.4f}")
print(f"  p-value     = {p_value:.4f}")
print(f"\nConclusion at α = 0.05:")
if p_value < 0.05:
    print("  ✓ REJECT null hypothesis - significant difference exists")
else:
    print("  ✗ FAIL TO REJECT null hypothesis - no significant difference")

# Chi-square test: Association between purchase frequency and category preference
contingency_table = pd.crosstab(survey['purchase_frequency'], survey['category_preference'])

print("📋 Contingency Table: Purchase Frequency × Category Preference")
print(contingency_table)

chi2, p_value, dof, expected = chi2_contingency(contingency_table)

print("\n\n🧪 Chi-Square Test of Independence")
print("=" * 60)
print(f"\nResults:")
print(f"  Chi-square statistic = {chi2:.4f}")
print(f"  Degrees of freedom   = {dof}")
print(f"  p-value             = {p_value:.4f}")
print(f"\nConclusion at α = 0.05:")
if p_value < 0.05:
    print("  ✓ REJECT null hypothesis - variables are DEPENDENT")
    print("  → Purchase frequency IS associated with category preference")
else:
    print("  ✗ FAIL TO REJECT null hypothesis - variables are INDEPENDENT")
    print("  → No significant association between purchase frequency and category")

# Test normality of satisfaction scores using multiple methods
data = survey['product_satisfaction']

print("📐 Normality Tests for Product Satisfaction Scores")
print("=" * 60)

# Shapiro-Wilk Test (best for n < 5000)
shapiro_stat, shapiro_p = stats.shapiro(data)
print(f"\n1. Shapiro-Wilk Test:")
print(f"   W-statistic = {shapiro_stat:.4f}")
print(f"   p-value     = {shapiro_p:.4f}")

# D'Agostino's K-squared Test
dagostino_stat, dagostino_p = stats.normaltest(data)
print(f"\n2. D'Agostino-Pearson Test:")
print(f"   K² statistic = {dagostino_stat:.4f}")
print(f"   p-value      = {dagostino_p:.4f}")

# Skewness and Kurtosis
skew = stats.skew(data)
kurt = stats.kurtosis(data)
print(f"\n3. Distribution Shape:")
print(f"   Skewness = {skew:.4f} ({'right-skewed' if skew > 0 else 'left-skewed' if skew < 0 else 'symmetric'})")
print(f"   Kurtosis = {kurt:.4f} ({'leptokurtic' if kurt > 0 else 'platykurtic' if kurt < 0 else 'mesokurtic'})")

print(f"\n📊 Conclusion:")
if shapiro_p > 0.05:
    print("   Data appears to be normally distributed (p > 0.05)")
else:
    print("   Data deviates significantly from normal distribution (p < 0.05)")

# Calculate 95% confidence intervals for satisfaction metrics
def confidence_interval(data, confidence=0.95):
    """Calculate confidence interval for mean"""
    n = len(data)
    mean = np.mean(data)
    se = stats.sem(data)  # Standard error of the mean
    h = se * stats.t.ppf((1 + confidence) / 2, n - 1)  # Margin of error
    return mean, mean - h, mean + h

print("📏 95% Confidence Intervals")
print("=" * 60)

metrics = {
    'Product Satisfaction': survey['product_satisfaction'],
    'Service Rating': survey['service_rating'],
    'Feedback Length': survey['feedback_length'],
    'Age': survey['age']
}

for name, data in metrics.items():
    mean, lower, upper = confidence_interval(data)
    print(f"\n{name}:")
    print(f"  Sample Mean: {mean:.2f}")
    print(f"  95% CI: [{lower:.2f}, {upper:.2f}]")
    print(f"  → We are 95% confident the true population mean")
    print(f"    falls between {lower:.2f} and {upper:.2f}")

# Generate comprehensive summary statistics by demographic groups
print("📊 SURVEY ANALYSIS SUMMARY")
print("=" * 70)

# Overall statistics
print(f"\n📋 Dataset Overview:")
print(f"   Total Respondents: {len(survey)}")
print(f"   Average Age: {survey['age'].mean():.1f} years")
print(f"   Gender Distribution: {dict(survey['gender'].value_counts())}")

# Key findings
print(f"\n🎯 Key Satisfaction Metrics:")
print(f"   Product Satisfaction: {survey['product_satisfaction'].mean():.2f}/5")
print(f"   Service Rating: {survey['service_rating'].mean():.2f}/5")
recommend_yes = (survey['would_recommend'] == 'Yes').sum()
print(f"   Would Recommend: {recommend_yes}/{len(survey)} ({100*recommend_yes/len(survey):.1f}%)")

# Category preferences
print(f"\n💻 Category Preferences:")
category_counts = survey['category_preference'].value_counts()
for category, count in category_counts.items():
    pct = (count / len(survey)) * 100
    print(f"   {category}: {count} ({pct:.1f}%)")

# Purchase patterns
print(f"\n⏱️ Purchase Frequency:")
frequency_counts = survey['purchase_frequency'].value_counts()
for freq, count in frequency_counts.items():
    pct = (count / len(survey)) * 100
    print(f"   {freq}: {count} ({pct:.1f}%)")

print("\n" + "=" * 70)