Program.cs

using AInDotNet.MLNET.CustomerChurn.Data;
using AInDotNet.MLNET.CustomerChurn.DataAnalysis;
using AInDotNet.MLNET.CustomerChurn.Evaluation;
using AInDotNet.MLNET.CustomerChurn.Model;
using AInDotNet.MLNET.CustomerChurn.Prediction;
using AInDotNet.MLNET.CustomerChurn.Training;
using Microsoft.ML;
using Microsoft.ML.Data;
const int randomSeed = 42;
const double finalTestFraction = 0.20;
const double validationFractionOfDevelopmentData = 0.25; // 25% of the remaining 80% produces a 60/20/20 train/validation/test split.
var dataPath = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Data", "WA_Fn-UseC_-Telco-Customer-Churn.csv"));

PrintPredictionContract();

//
// Step 1: Profile the raw data before modeling.
//
CustomerChurnDataProfiler.Profile(dataPath);

//
// Step 2: Load the data into strongly typed C# objects.
//
var customers = CustomerChurnDataLoader.Load(dataPath);

Console.WriteLine();
Console.WriteLine($"Loaded {customers.Count:N0} customers for modeling.");

//
// Step 3: Convert the data to an ML.NET IDataView.
//
var mlContext = new MLContext(seed: randomSeed);

var data = mlContext.Data.LoadFromEnumerable(customers);

//
// Keep the final test population untouched while features, trainer, and threshold are selected on validation data.
// Reusing the final test set during those choices would make the reported final result optimistically biased.
//
var developmentAndTestSplit = mlContext.Data.TrainTestSplit(data, testFraction: finalTestFraction, seed: randomSeed);
var trainingAndValidationSplit = mlContext.Data.TrainTestSplit(developmentAndTestSplit.TrainSet, testFraction: validationFractionOfDevelopmentData, seed: randomSeed);
var trainingData = trainingAndValidationSplit.TrainSet;
var validationData = trainingAndValidationSplit.TestSet;
var finalTestData = developmentAndTestSplit.TestSet;

//
// Show class balance in the actual train/test sets.
//
PrintSplitSummary(mlContext, trainingData, validationData, finalTestData);


//
// Run A
//
// Question:
// How well can we predict churn using only four numeric features?
//
PrintExperimentQuestion("Run A", "How well can we predict churn using only four numeric customer features?");

BinaryClassificationMetrics runAMetrics;

var runAModel =
    BaselineClassifierTrainer.Train(
        mlContext,
        trainingData,
        validationData,
        FeatureSet.BasicNumeric,
        "A",
        "Basic numeric features",
        out runAMetrics);

//
// Run B
//
// Question:
// Does adding obvious business-context fields improve the model?
//
PrintExperimentQuestion("Run B", "Does adding contract, internet service, payment method, and paperless billing improve churn prediction?");

BinaryClassificationMetrics runBMetrics;

var runBModel = BaselineClassifierTrainer.Train(
        mlContext,
        trainingData,
        validationData,
        FeatureSet.Expanded,
        "B",
        "Numeric + selected categorical features",
        out runBMetrics);

//PrintMetricChange("A", runAMetrics, "B", runBMetrics);

//
// Run C
//
// Question:
// Does adding the full customer and service context improve the model further?
//
PrintExperimentQuestion("Run C", "Does adding the full set of customer, service, contract, and billing features improve churn prediction further?");

BinaryClassificationMetrics runCMetrics;

var runCModel = BaselineClassifierTrainer.Train(
        mlContext,
        trainingData,
        validationData,
        FeatureSet.Full,
        "C",
        "Full numeric + categorical features",
        out runCMetrics);

//PrintMetricChange("B", runBMetrics, "C", runCMetrics);

//
// Run D
//
// Question:
// After improving the features, can AutoML find
// a better binary-classification algorithm?
//
PrintExperimentQuestion("Run D", "After improving the features, can AutoML find a better binary-classification algorithm and hyperparameters?");

var runD = await AutoMLClassifierTrainer.TrainAsync(mlContext, trainingData, validationData);

var runDModel = runD.Model;

var runDMetrics = runD.Metrics;

//PrintMetricChange("C", runCMetrics, "D", runDMetrics);


//
// Run E
//
// Question:
// Can we identify more churners by changing
// the classification threshold?
//
var selectedThreshold = ThresholdTuningEvaluator.SelectThreshold(mlContext, runDModel, validationData);

// Only now, after model and threshold selection are complete, evaluate once on the untouched final test population.
var finalTestPredictions = runDModel.Transform(finalTestData);
var finalTestMetrics = mlContext.BinaryClassification.Evaluate(finalTestPredictions, labelColumnName: nameof(CustomerChurnData.Churn), scoreColumnName: "Score", probabilityColumnName: "Probability", predictedLabelColumnName: "PredictedLabel");
BinaryClassificationEvaluator.PrintMetrics(finalTestMetrics, "Final Test Evaluation at ML.NET's Default 0.50 Threshold");
BinaryClassificationEvaluator.AnalyzePredictions(mlContext, finalTestPredictions, "Final Test Classification Analysis");
ThresholdTuningEvaluator.EvaluateSelectedThreshold(mlContext, runDModel, finalTestData, selectedThreshold);


//
// Save the winning Run D model.
//
var modelPath = CustomerChurnPredictor.SaveModel(mlContext, runDModel, trainingData.Schema);

//
// Reload the saved model and score
// one hypothetical customer.
//
CustomerChurnPredictor.ScoreHypotheticalCustomer(mlContext, modelPath, selectedThreshold);

//
// Final comparison
//
PrintFinalComparison(runAMetrics, runBMetrics, runCMetrics, runDMetrics);

Console.WriteLine();
Console.WriteLine("WHAT WE LEARNED");
Console.WriteLine("===============");
Console.WriteLine("1. Churn is an imbalanced classification problem.");
Console.WriteLine("2. Accuracy alone can be misleading.");
Console.WriteLine("3. Precision and recall measure very different kinds of success.");
Console.WriteLine("4. Adding features does not automatically make every metric better.");
Console.WriteLine($"5. AutoML selected {runD.TrainerName}, which produced the strongest validation AUC.");
Console.WriteLine("6. The default 0.50 classification threshold is not automatically the best business threshold.");
Console.WriteLine("7. Lowering the threshold identified substantially more churners.");
Console.WriteLine($"8. A {selectedThreshold:F2} threshold produced the best F1 score on validation data; final-test performance was reported separately.");
Console.WriteLine("9. The right threshold depends on the business cost of false positives versus false negatives.");
Console.WriteLine("10. Predictive AI is not finished when the model is trained. The prediction must be converted into a business decision.");

Console.WriteLine();
Console.WriteLine("Done.");

static void PrintExperimentQuestion(string runName, string question)
{
    Console.WriteLine();
    Console.WriteLine($"{runName} Experiment Question");
    Console.WriteLine(new string('-', $"{runName} Experiment Question".Length));

    Console.WriteLine(question);
    Console.WriteLine();
}

static void PrintPredictionContract()
{
    Console.WriteLine("PREDICTION CONTRACT");
    Console.WriteLine("===================");
    Console.WriteLine("Business question: Which current customers are at elevated risk of churning?");
    Console.WriteLine("Model output:      A churn score/probability and a threshold-based classification.");
    Console.WriteLine("Historical label:  Churn = Yes or No.");
    Console.WriteLine("Prediction point:  A current customer snapshot, before the retention decision.");
    Console.WriteLine("Action:            Prioritize selected customers for a retention workflow.");
    Console.WriteLine("Teaching caveat:   This dataset does not define a future prediction horizon, so it cannot prove prospective performance.");
    Console.WriteLine();
}

static void PrintSplitSummary(MLContext mlContext, IDataView trainData, IDataView validationData, IDataView testData)
{
    var trainingRows =
        mlContext.Data
            .CreateEnumerable<CustomerChurnData>(
                trainData,
                reuseRowObject: false)
            .ToList();

    var validationRows = mlContext.Data.CreateEnumerable<CustomerChurnData>(validationData, reuseRowObject: false).ToList();
    var testRows = mlContext.Data.CreateEnumerable<CustomerChurnData>(testData, reuseRowObject: false).ToList();

    Console.WriteLine();
    Console.WriteLine("Training / Validation / Final Test Split");
    Console.WriteLine("========================================");
    Console.WriteLine();

    PrintClassDistribution("Training", trainingRows);

    Console.WriteLine();

    PrintClassDistribution("Validation (used for experiment and threshold selection)", validationRows);
    Console.WriteLine();
    PrintClassDistribution("Final Test (untouched until all choices are complete)", testRows);
}

static void PrintClassDistribution(string label, IReadOnlyList<CustomerChurnData> customers)
{
    var churned = customers.Count(x => x.Churn);

    var stayed = customers.Count - churned;

    var churnRate =
        customers.Count == 0
            ? 0
            : (double)churned /
              customers.Count *
              100.0;

    Console.WriteLine(label);
    Console.WriteLine(new string('-', label.Length));

    Console.WriteLine($"Customers: {customers.Count:N0}");

    Console.WriteLine($"Stayed:    {stayed:N0}");

    Console.WriteLine($"Churned:   {churned:N0}");

    Console.WriteLine($"Churn Rate:{churnRate,7:F1}%");
}

static void PrintMetricChange(string previousRun, BinaryClassificationMetrics previous, string currentRun, BinaryClassificationMetrics current)
{
    Console.WriteLine();
    Console.WriteLine($"Change from Run {previousRun} to Run {currentRun}");

    Console.WriteLine(new string('-', $"Change from Run {previousRun} to Run {currentRun}".Length));

    Console.WriteLine($"Accuracy:  {previous.Accuracy:P1} -> {current.Accuracy:P1}");

    Console.WriteLine($"Precision: {previous.PositivePrecision:P1} -> {current.PositivePrecision:P1}");

    Console.WriteLine($"Recall:    {previous.PositiveRecall:P1} -> {current.PositiveRecall:P1}");

    Console.WriteLine($"F1:        {previous.F1Score:F3} -> {current.F1Score:F3}");

    Console.WriteLine($"AUC:       {previous.AreaUnderRocCurve:F3} -> {current.AreaUnderRocCurve:F3}");
}

static void PrintFinalComparison(BinaryClassificationMetrics runA, BinaryClassificationMetrics runB, BinaryClassificationMetrics runC, BinaryClassificationMetrics runD)
{
    Console.WriteLine();
    Console.WriteLine("VALIDATION EXPERIMENT COMPARISON");
    Console.WriteLine("================================");

    Console.WriteLine(
        $"{"Run",-5}" +
        $"{"Accuracy",12}" +
        $"{"Precision",12}" +
        $"{"Recall",12}" +
        $"{"F1",10}" +
        $"{"AUC",10}");

    Console.WriteLine(new string('-', 61));

    PrintComparisonRow("A", runA);

    PrintComparisonRow("B", runB);

    PrintComparisonRow("C", runC);
    PrintComparisonRow("D", runD);

    Console.WriteLine();
    Console.WriteLine("Important:");
    Console.WriteLine("Do not choose the best churn model using accuracy alone.");
    Console.WriteLine("Recall, precision, F1, AUC, and the business cost of");
    Console.WriteLine("false positives and false negatives all matter.");
}

static void PrintComparisonRow(string run, BinaryClassificationMetrics metrics)
{
    Console.WriteLine(
        $"{run,-5}" +
        $"{metrics.Accuracy,12:P1}" +
        $"{metrics.PositivePrecision,12:P1}" +
        $"{metrics.PositiveRecall,12:P1}" +
        $"{metrics.F1Score,10:F3}" +
        $"{metrics.AreaUnderRocCurve,10:F3}");
}