GAI: Generalized Alpha-Investing¶

from online_fdr.investing.alpha.alpha import Gai

# Create GAI instance
gai = Gai(alpha=0.05, wealth=0.025)

# Test individual p-values
p_values = [0.001, 0.15, 0.03, 0.8, 0.02, 0.45, 0.006]

print("Generalized Alpha-Investing:")
discoveries = []

for i, p_value in enumerate(p_values):
    decision = gai.test_one(p_value)

    if decision:
        discoveries.append(i + 1)
        print(f"✓ Test {i+1}: p={p_value:.3f} → DISCOVERY!")
    else:
        print(f"  Test {i+1}: p={p_value:.3f} → no rejection")

print(f"\nTotal discoveries: {len(discoveries)}")
print(f"Discovery indices: {discoveries}")

def demonstrate_gai_mechanism():
    """Show how GAI differs from fixed-level testing."""

    print("GAI vs Fixed-Level Testing:")
    print("=" * 35)

    test_sequence = [0.001, 0.8, 0.02, 0.9, 0.005, 0.7, 0.015]

    # GAI with dynamic thresholds
    gai = Gai(alpha=0.1, wealth=0.05)  # Higher values for visibility

    print("GAI (Dynamic Thresholds):")
    gai_discoveries = 0

    for i, p_val in enumerate(test_sequence, 1):
        # Calculate threshold before testing
        gai.num_test += 1
        threshold = gai.calc_alpha_t()

        decision = p_val <= threshold

        if decision:
            gai_discoveries += 1
            gai.candidates.append(p_val <= gai.alpha0)  # Assuming p_val as candidate check
            gai.reject_idx.append(gai.num_test)

        print(f"Test {i}: p={p_val:.3f}, threshold={threshold:.6f} → {'REJECT' if decision else 'ACCEPT'}")

    # Fixed-level testing for comparison
    print(f"\nFixed-Level (α=0.05):")
    fixed_discoveries = 0

    for i, p_val in enumerate(test_sequence, 1):
        decision = p_val <= 0.05
        if decision:
            fixed_discoveries += 1
        print(f"Test {i}: p={p_val:.3f}, threshold=0.050000 → {'REJECT' if decision else 'ACCEPT'}")

    print(f"\nComparison:")
    print(f"GAI discoveries: {gai_discoveries}")
    print(f"Fixed-level discoveries: {fixed_discoveries}")
    print(f"Power advantage: {gai_discoveries - fixed_discoveries}")

demonstrate_gai_mechanism()

def gai_with_prior_knowledge():
    """Demonstrate how GAI can incorporate domain knowledge."""

    print("GAI with Prior Knowledge:")
    print("=" * 30)

    # Simulate a scenario where you expect more promising tests later
    # (e.g., genomics where genes are ordered by biological relevance)

    # Early tests: mostly null
    early_tests = [0.8, 0.7, 0.9, 0.6, 0.75]

    # Later tests: mix of null and alternative  
    later_tests = [0.001, 0.02, 0.8, 0.005, 0.03, 0.9, 0.007]

    # Conservative start to preserve wealth for later promising tests
    gai = Gai(alpha=0.05, wealth=0.01)  # Lower initial wealth

    print("Early phase (expected mostly nulls):")
    early_discoveries = 0

    for i, p_val in enumerate(early_tests, 1):
        decision = gai.test_one(p_val)
        if decision:
            early_discoveries += 1
            print(f"✓ Test {i}: p={p_val:.3f} → DISCOVERY")
        else:
            print(f"  Test {i}: p={p_val:.3f} → no rejection (conservative)")

    print(f"\nTransition to promising region...")
    print("Later phase (expected more alternatives):")

    later_discoveries = 0
    for i, p_val in enumerate(later_tests, len(early_tests) + 1):
        decision = gai.test_one(p_val)
        if decision:
            later_discoveries += 1
            print(f"✓ Test {i}: p={p_val:.3f} → DISCOVERY")
        else:
            print(f"  Test {i}: p={p_val:.3f} → no rejection")

    print(f"\nResults:")
    print(f"Early discoveries: {early_discoveries}")
    print(f"Later discoveries: {later_discoveries}")
    print(f"Total discoveries: {early_discoveries + later_discoveries}")
    print("GAI preserved wealth for the promising region!")

gai_with_prior_knowledge()

from online_fdr.investing.saffron.saffron import Saffron
from online_fdr.investing.lord.three import LordThree

def compare_alpha_investing_family():
    """Compare different alpha-investing approaches."""

    print("Alpha-Investing Family Comparison:")
    print("=" * 40)

    # Test sequence with varied difficulty
    p_values = [0.002, 0.8, 0.01, 0.9, 0.005, 0.7, 0.03, 0.6, 0.008]

    # Different alpha-investing approaches
    methods = {
        'GAI': Gai(alpha=0.05, wealth=0.025),
        'SAFFRON': Saffron(alpha=0.05, wealth=0.025, lambda_=0.5),
        'LORD 3': LordThree(alpha=0.05, wealth=0.025, reward=0.025)
    }

    results = {}

    for method_name, method in methods.items():
        decisions = [method.test_one(p) for p in p_values]
        discoveries = sum(decisions)
        discovery_indices = [i+1 for i, d in enumerate(decisions) if d]

        results[method_name] = {
            'discoveries': discoveries,
            'indices': discovery_indices
        }

        print(f"{method_name:>8}: {discoveries} discoveries at positions {discovery_indices}")

    return results

compare_alpha_investing_family()

from online_fdr.utils.generation import DataGenerator, GaussianLocationModel

def simulate_ab_testing_with_gai():
    """Simulate GAI in an industrial A/B testing environment."""

    print("Industrial A/B Testing with GAI:")
    print("=" * 36)

    # Simulate A/B testing scenario:
    # - Many tests run simultaneously
    # - Most are null (no real effect)
    # - Some have real but small effects
    # - Occasional large effects

    dgp = GaussianLocationModel(alt_mean=1.5, alt_std=1.0, one_sided=True)
    generator = DataGenerator(n=200, pi0=0.95, dgp=dgp)  # 95% nulls (realistic)

    gai = Gai(alpha=0.05, wealth=0.025)

    true_positives = 0
    false_positives = 0
    test_count = 0

    print("Running A/B tests sequentially...")

    # Simulate first 100 tests
    for i in range(100):
        p_value, is_alternative = generator.sample_one()
        decision = gai.test_one(p_value)
        test_count += 1

        if decision:
            if is_alternative:
                true_positives += 1
                result_type = "TRUE effect ✓"
            else:
                false_positives += 1
                result_type = "FALSE alarm ✗"

            # Show significant results
            effect_type = "REAL" if is_alternative else "NULL"
            print(f"Test {i+1:3d}: p={p_value:.4f} ({effect_type}) → SIGNIFICANT ({result_type})")

    # Calculate business metrics
    total_discoveries = true_positives + false_positives
    empirical_fdr = false_positives / max(total_discoveries, 1)
    power = true_positives / max(sum(generator.sample_one()[1] for _ in range(100)), 1)  # Approximate

    print(f"\nA/B Testing Campaign Results:")
    print(f"Total tests run: {test_count}")
    print(f"Significant effects found: {total_discoveries}")
    print(f"True discoveries (real effects): {true_positives}")
    print(f"False alarms: {false_positives}")
    print(f"False Discovery Rate: {empirical_fdr:.3f}")
    print(f"Target FDR: {gai.alpha0}")
    print(f"FDR controlled: {'✓' if empirical_fdr <= gai.alpha0 else '✗'}")

    # Business interpretation
    print(f"\nBusiness Impact:")
    if true_positives > 0:
        print(f"✓ Found {true_positives} real improvements to implement")
    if false_positives > 0:
        print(f"⚠ {false_positives} false alarms avoided implementing bad changes")

    efficiency = true_positives / max(total_discoveries, 1)
    print(f"Discovery efficiency: {efficiency:.1%}")

simulate_ab_testing_with_gai()