Program.cs

using System.Text.Json;
using AInDotNet.MLNET.CreditCardFraud.Data;
using AInDotNet.MLNET.CreditCardFraud.DataAnalysis;
using AInDotNet.MLNET.CreditCardFraud.Evaluation;
using AInDotNet.MLNET.CreditCardFraud.Model;
using AInDotNet.MLNET.CreditCardFraud.Training;
using Microsoft.ML;
using Microsoft.ML.Data;

const int randomSeed = 42;
const double validationAndTestFraction = 0.40;
const double finalTestFractionOfRemainder = 0.50;
const uint autoMlTimeBudgetSeconds = 60;
const double selectionFalsePositiveCost = 25.00;
const double selectionMissedFraudMultiplier = 1.00;

var projectDirectory = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", ".."));
var dataPath = Path.Combine(projectDirectory, "Data", "creditcard.csv");
var artifactDirectory = Path.Combine(projectDirectory, "ModelArtifacts");

CreditCardFraudDataProfiler.Profile(dataPath);

var originalRecords = CreditCardFraudDataLoader.Load(dataPath);

// Keep the first occurrence of each exact observation. Allowing identical observations to cross data splits could make validation and final-test performance look better than performance on genuinely unseen transactions.
var records = originalRecords.DistinctBy(row => (
    row.Time, row.V1, row.V2, row.V3, row.V4, row.V5, row.V6, row.V7, row.V8, row.V9, row.V10, row.V11, row.V12, row.V13, row.V14,
    row.V15, row.V16, row.V17, row.V18, row.V19, row.V20, row.V21, row.V22, row.V23, row.V24, row.V25, row.V26, row.V27, row.V28, row.Amount, row.Label)).ToList();

var duplicateCount = originalRecords.Count - records.Count;

Console.WriteLine("Modeling Population Preparation");
Console.WriteLine("-------------------------------");
Console.WriteLine($"Original rows:         {originalRecords.Count,10:N0}");
Console.WriteLine($"Exact duplicates removed: {duplicateCount,6:N0}");
Console.WriteLine($"Rows used for modeling:   {records.Count,6:N0}");
Console.WriteLine();
Console.WriteLine("Exact duplicate observations are removed before splitting so copies of");
Console.WriteLine("the same transaction cannot appear in both training and evaluation data.");
Console.WriteLine();

if (records.Count == 0)
    throw new InvalidOperationException("No valid transactions remain after duplicate removal.");

var mlContext = new MLContext(seed: randomSeed);
var data = mlContext.Data.LoadFromEnumerable(records);

// Model and policy selection must not use the final test population. The first split reserves 40%, and the second divides that remainder equally into validation and test sets.
var trainingAndRemainder = mlContext.Data.TrainTestSplit(data, testFraction: validationAndTestFraction, seed: randomSeed);
var validationAndTest = mlContext.Data.TrainTestSplit(trainingAndRemainder.TestSet, testFraction: finalTestFractionOfRemainder, seed: randomSeed);
var trainingData = trainingAndRemainder.TrainSet;
var validationData = validationAndTest.TrainSet;
var testData = validationAndTest.TestSet;

PrintPopulation("Training", mlContext, trainingData);
PrintPopulation("Validation", mlContext, validationData);
PrintPopulation("Final test", mlContext, testData);

MajorityClassBaseline.Run(mlContext, validationData);
var runBModel = FastTreeFraudTrainer.Run(mlContext, trainingData, validationData);
var runCModel = UndersampledFastTreeFraudTrainer.Run(mlContext, trainingData, validationData, legitimateToFraudRatio: 10);
var runDModel = AutoMLFraudTrainer.Run(mlContext, trainingData, validationData, maxExperimentTimeInSeconds: autoMlTimeBudgetSeconds);
var thresholdSelection = ThresholdTuningExperiment.Run(mlContext, runDModel, validationData);

FraudBusinessCostSimulator.RunSensitivityAnalysis(mlContext, validationData, runBModel, runCModel, runDModel, thresholdSelection.CandidateThresholds);
var selectedPolicy = FraudBusinessCostSimulator.SelectPolicy(mlContext, validationData, runBModel, runCModel, runDModel, thresholdSelection.CandidateThresholds, selectionFalsePositiveCost, selectionMissedFraudMultiplier);

Console.WriteLine();
Console.WriteLine("Run G - Final Unbiased Test Evaluation");
Console.WriteLine("======================================");
Console.WriteLine();
Console.WriteLine("The model and threshold were selected using validation data.");
Console.WriteLine("This is the first and only evaluation of that policy on the untouched final test set.");
Console.WriteLine();

FraudBusinessCostSimulator.EvaluateFinalPolicy(mlContext, testData, selectedPolicy, selectionFalsePositiveCost, selectionMissedFraudMultiplier);

Directory.CreateDirectory(artifactDirectory);

var modelPath = Path.Combine(artifactDirectory, "CreditCardFraudModel.zip");
var policyPath = Path.Combine(artifactDirectory, "CreditCardFraudDecisionPolicy.json");
var serializedPolicy = new SavedDecisionPolicy
{
    RunName = selectedPolicy.RunName,
    Threshold = selectedPolicy.Threshold,
    FalsePositiveCost = selectionFalsePositiveCost,
    MissedFraudLossMultiplier = selectionMissedFraudMultiplier
};

mlContext.Model.Save(selectedPolicy.Model, trainingData.Schema, modelPath);
File.WriteAllText(policyPath, JsonSerializer.Serialize(serializedPolicy, new JsonSerializerOptions { WriteIndented = true }));

Console.WriteLine();
Console.WriteLine($"Saved model:  {modelPath}");
Console.WriteLine($"Saved policy: {policyPath}");

VerifySavedArtifacts(mlContext, selectedPolicy.Model, records[0], modelPath, policyPath, serializedPolicy);

static void PrintPopulation(string name, MLContext mlContext, IDataView data)
{
    var rows = mlContext.Data.CreateEnumerable<CreditCardFraudData>(data, reuseRowObject: false).ToList();
    var fraudCount = rows.Count(row => row.Label);
    var fraudRate = rows.Count == 0 ? 0 : (double)fraudCount / rows.Count;
    Console.WriteLine($"{name,-12} rows: {rows.Count,7:N0}   fraud: {fraudCount,4:N0} ({fraudRate:P3})");
}

static void VerifySavedArtifacts(MLContext mlContext, ITransformer originalModel, CreditCardFraudData verificationRecord, string modelPath, string policyPath, SavedDecisionPolicy expectedPolicy)
{
    var loadedModel = mlContext.Model.Load(modelPath, out _);
    var loadedPolicy = JsonSerializer.Deserialize<SavedDecisionPolicy>(File.ReadAllText(policyPath)) ?? throw new InvalidOperationException("The saved decision-policy file could not be deserialized.");
    var verificationData = mlContext.Data.LoadFromEnumerable(new[] { verificationRecord });
    var originalProbability = GetProbability(mlContext, originalModel, verificationData);
    var loadedProbability = GetProbability(mlContext, loadedModel, verificationData);
    var sameProbability = Math.Abs(originalProbability - loadedProbability) <= 0.000001f;
    var samePolicy = loadedPolicy.RunName == expectedPolicy.RunName && Math.Abs(loadedPolicy.Threshold - expectedPolicy.Threshold) <= 0.000001 &&
        Math.Abs(loadedPolicy.FalsePositiveCost - expectedPolicy.FalsePositiveCost) <= 0.000001 && Math.Abs(loadedPolicy.MissedFraudLossMultiplier - expectedPolicy.MissedFraudLossMultiplier) <= 0.000001;

    if (!sameProbability || !samePolicy)
        throw new InvalidOperationException("The saved model or decision policy did not reproduce the selected in-memory artifacts.");

    Console.WriteLine();
    Console.WriteLine("Saved Artifact Verification");
    Console.WriteLine("---------------------------");
    Console.WriteLine("Model reload:           PASS");
    Console.WriteLine("Prediction consistency: PASS");
    Console.WriteLine("Policy reload:          PASS");
}

static float GetProbability(MLContext mlContext, ITransformer model, IDataView data)
{
    var scoredData = model.Transform(data);

    if (scoredData.Schema.GetColumnOrNull("Probability") == null)
        throw new InvalidOperationException("The saved model did not produce the Probability column required by the decision policy.");

    return mlContext.Data.CreateEnumerable<FraudProbability>(scoredData, reuseRowObject: false).Single().Probability;
}

internal sealed class FraudProbability
{
    [ColumnName("Probability")]
    public float Probability { get; set; }
}

internal sealed class SavedDecisionPolicy
{
    public string RunName { get; set; } = string.Empty;
    public double Threshold { get; set; }
    public double FalsePositiveCost { get; set; }
    public double MissedFraudLossMultiplier { get; set; }
}