working on models for real-world data

Author: pat-alt

docs/src/paper/Project.toml

@@ -7,6 +7,7 @@ Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0"
 LaplaceRedux = "c52c1a26-f7c5-402b-80be-ba1e638ad478"
 MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
+MLJModels = "d491faf4-2d78-11e9-2867-c94bc002c0b7"
 MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 RCall = "6f49c342-dc21-5d91-9882-a32aef131414"

docs/src/paper/data_preprocessing/_real_world_data.qmd

@@ -27,70 +27,103 @@ if not os.path.isdir(os.path.join(data_path,"raw")):
 df.to_csv(os.path.join(data_path,"raw/cal_housing.csv"), index=False)
 ```
 
-Loading the data into Julia session:
+Loading the data into Julia session. 
 
 ```{julia}
 df = CSV.read(joinpath(data_path, "raw/cal_housing.csv"), DataFrame)
-# Features:
-X = Matrix(df[:,Not(:target)])
-dt = StatsBase.fit(ZScoreTransform, X, dims=1)
-StatsBase.transform!(dt, X)
+# # Features:
+# X = Matrix(df[:,Not(:target)])
+# dt = StatsBase.fit(ZScoreTransform, X, dims=1)
+# StatsBase.transform!(dt, X)
+# df = DataFrame(X,:auto)
 # Target:
 y = df.target
-y = Float64.(y .>= median(y)); # binary target
+y = Float64.(y .>= median(y)); # binary target (positive outcome)
 # Data:
-df = DataFrame(X,:auto)
 df.target = y
 ```
 
+Random undersampling to balance the data:
+
 ```{julia}
-using MLUtils: undersample
-# Make DataFrames.jl work
-MLUtils.getobs(data::DataFrame, i) = data[i,:]
-MLUtils.numobs(data::DataFrame) = nrow(data)
 df_balanced = getobs(undersample(df, df.target;shuffle=true))[1]
 ```
 
+All features are continuous:
+
+```{julia}
+schema(df_balanced)
+```
+
+Turning the data into `CounterfactualData`:
+
 ```{julia}
-CSV.write(joinpath(data_path, "cal_housing.csv"), df_balanced)
+X = Matrix(df_balanced[:,Not(:target)])
+X = permutedims(X)
+y = permutedims(df_balanced.target)
+data = CounterfactualData(X,y)
+```
+
+Saving the data:
+
+```{julia}
+CSV.write(joinpath(data_path, "cal_housing.csv"), df_balanced)              # binary file
+Serialization.serialize(joinpath(data_path,"cal_housing.jls"), data)        # CounterfactualData
 ```
 
 
 ## Give Me Some Credit
 
+Loading and basic preprocessing:
+
 ```{julia}
 df = CSV.read(joinpath(data_path, "raw/cs-training.csv"), DataFrame)
 select!(df, Not([:Column1]))
 rename!(df, :SeriousDlqin2yrs => :target)
 mapcols!(x -> [ifelse(x_=="NA", missing, x_) for x_ in x], df)
 dropmissing!(df)
 mapcols!(x -> eltype(x) <: AbstractString ? parse.(Int, x) : x, df)
-# Features:
-X = Matrix(df[:,Not(:target)])
-dt = StatsBase.fit(ZScoreTransform, X, dims=1)
-StatsBase.transform!(dt, X)
+# # Features:
+# X = Matrix(df[:,Not(:target)])
+# dt = StatsBase.fit(ZScoreTransform, X, dims=1)
+# StatsBase.transform!(dt, X)
+# df = DataFrame(X,:auto)
 # Target:
-y = df.target
-# Data:
-df = DataFrame(X,:auto)
-df.target = y
+df.target .= map(y -> y == 0 ? 1 : 0, df.target)        # postive outcome = no delinquency
 ```
 
+Balancing:
+
 ```{julia}
-using MLUtils
-using MLUtils: undersample
-# Make DataFrames.jl work
-MLUtils.getobs(data::DataFrame, i) = data[i,:]
-MLUtils.numobs(data::DataFrame) = nrow(data)
 df_balanced = getobs(undersample(df, df.target;shuffle=true))[1]
 ```
 
+All features are continuous:
+
 ```{julia}
-CSV.write(joinpath(data_path, "gmsc.csv"), df_balanced)
+schema(df_balanced)
+```
+
+Turning the data into `CounterfactualData`:
+
+```{julia}
+X = Matrix(df_balanced[:,Not(:target)])
+X = permutedims(X)
+y = permutedims(df_balanced.target)
+data = CounterfactualData(X,y)
+```
+
+Saving:
+
+```{julia}
+CSV.write(joinpath(data_path, "gmsc.csv"), df_balanced)         # binary file
+Serialization.serialize(joinpath(data_path,"gmsc.jls"), data)   # CounterfactualData
 ```
 
 ## UCI Credit Card Default
 
+Loading and basic preprocessing:
+
 ```{julia}
 df = CSV.read(joinpath(data_path, "raw/UCI_Credit_Card.csv"), DataFrame)
 select!(df, Not([:ID]))
@@ -100,17 +133,57 @@ df.SEX = categorical(df.SEX)
 df.EDUCATION = categorical(df.EDUCATION)
 df.MARRIAGE = categorical(df.MARRIAGE)
 mapcols!(x -> eltype(x) <: AbstractString ? parse.(Int, x) : x, df)
+df.target .= map(y -> y == 0 ? 1 : 0, df.target)        # postive outcome = no default
 ```
 
+Balancing:
+
 ```{julia}
-# Make DataFrames.jl work
-MLUtils.getobs(data::DataFrame, i) = data[i,:]
-MLUtils.numobs(data::DataFrame) = nrow(data)
 df_balanced = getobs(undersample(df, df.target;shuffle=true))[1]
 ```
 
+**Not** all features are continuous:
+
+```{julia}
+schema(df_balanced)
+```
+
+One-hot encoding:
+
+```{julia}
+hot = OneHotEncoder()
+mach = MLJBase.fit!(machine(hot, df_balanced))
+df_balanced = MLJBase.transform(mach, df_balanced)
+schema(df_balanced)
+```
+
+Categorical indices:
+
+```{julia}
+features_categorical = [
+    [2,3],
+    collect(4:10),
+    collect(11:14)
+]
+```
+
+Preparing for use with `CounterfactualExplanations.jl`:
+
+```{julia}
+X = Matrix(df_balanced[:,Not(:target)])
+X = permutedims(X)
+y = permutedims(df_balanced.target)
+data = CounterfactualData(
+    X, y;
+    features_categorical = features_categorical
+)
+```
+
+Saving:
+
 ```{julia}
-CSV.write(joinpath(data_path, "credit_default.csv"), df_balanced)
+CSV.write(joinpath(data_path, "credit_default.csv"), df_balanced)         # binary file
+Serialization.serialize(joinpath(data_path,"credit_default.jls"), data)   # CounterfactualData
 ```
 
 

docs/src/paper/experiments/_real_world.qmd

@@ -6,12 +6,13 @@
 include("docs/src/paper/setup.jl")
 eval(setup)
 output_path = output_dir("real_world")
-www_path = www_dir("real_world");
+www_path = www_dir("real_world")
+data_path = data_dir("real_world")
 ```
 
 ```{julia}
 max_obs = 2500
-data_sets = AlgorithmicRecourseDynamics.Data.load_real_world(max_obs)
+data_sets = AlgorithmicRecourseDynamics.Data.load_real_world(max_obs; data_dir=data_path)
 choices = [
     :cal_housing, 
     :credit_default, 
@@ -22,13 +23,13 @@ data_sets = filter(p -> p[1] in choices, data_sets)
 
 ```{julia}
 using CounterfactualExplanations.DataPreprocessing: unpack
-bs = 50
+bs = 500
 function data_loader(data::CounterfactualData)
     X, y = unpack(data)
     data = Flux.DataLoader((X,y),batchsize=bs)
     return data
 end
-model_params = (batch_norm=false,n_hidden=32,n_layers=3,dropout=true,p_dropout=0.25)
+model_params = (batch_norm=false,n_hidden=64,n_layers=3,dropout=true,p_dropout=0.5)
 ```
 
 
@@ -62,9 +63,8 @@ n_rounds = 50
 evaluate_every = Int(round(n_rounds/n_evals))
 n_folds = 5
 n_samples = 10000
-T = 250
+T = 100
 generative_model_params = (epochs=250, latent_dim=8)
-using Serialization
 results = run_experiments(
     experiments;
     save_path=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T, n_samples=n_samples,
@@ -76,7 +76,6 @@ Serialization.serialize(joinpath(output_path,"results.jls"),results)
 #### Plots
 
 ```{julia}
-using Serialization
 results = Serialization.deserialize(joinpath(output_path,"results.jls"))
 ```
 

docs/src/paper/setup.jl

@@ -17,7 +17,9 @@ setup = quote
     using LaplaceRedux
     using Markdown
     using MLJBase
+    using MLJModels: OneHotEncoder
     using MLUtils
+    using MLUtils: undersample
     using Plots
     using Random
     using RCall
@@ -29,4 +31,8 @@ setup = quote
     theme(:wong)
     include("docs/src/utils.jl")    # some helper functions
 
+    # Make DataFrames.jl work
+    MLUtils.getobs(data::DataFrame, i) = data[i, :]
+    MLUtils.numobs(data::DataFrame) = nrow(data)
+
 end

src/base.jl

@@ -175,7 +175,10 @@ function set_up_experiment(
 
     # Pretrain:
     if !isnothing(pre_train_models)
-        map!(model -> Models.train(model, data_train; n_epochs=pre_train_models, kwargs...), values(models))
+        for (key, model) in models
+            @info "Training $key"
+            Models.train(model, data_train; n_epochs=pre_train_models, kwargs...)
+        end
     end
 
     experiment = Experiment(data_train, data_test, target, models, deepcopy(generators), num_counterfactuals)
@@ -218,7 +221,11 @@ function set_up_experiments(
         kwargs...
     )
 
-    experiments = Dict(key => set_up_single(data) for (key, data) in catalogue)
+    experiments = Dict{Symbol, Experiment}()
+    for (key, data) in catalogue
+        @info "Setting up $(key)"
+        experiments[key] = set_up_single(data)
+    end
 
     return experiments
 end

src/data/functions.jl

@@ -2,42 +2,41 @@ using LazyArtifacts
 using CounterfactualExplanations.DataPreprocessing: CounterfactualData
 using CSV
 using DataFrames
+using Serialization
 using StatsBase
 
-function load_synthetic(max_obs::Union{Nothing, Int}=nothing)
-    data_dir = joinpath(artifact"data","data/synthetic")
+function load_synthetic(max_obs::Union{Nothing,Int}=nothing)
+    data_dir = joinpath(artifact"data", "data/synthetic")
     files = readdir(data_dir)
-    files = files[contains.(files,".csv")]
+    files = files[contains.(files, ".csv")]
     data = map(files) do file
         df = CSV.read(joinpath(data_dir, file), DataFrame)
-        X = convert(Matrix, hcat(df.x1,df.x2)')
+        X = convert(Matrix, hcat(df.x1, df.x2)')
         y = convert(Matrix, df.target')
-        data = CounterfactualData(X,y)
+        data = CounterfactualData(X, y)
         if !isnothing(max_obs)
             n_classes = length(unique(y))
-            data = undersample(data, Int(round(max_obs/n_classes)))
+            data = undersample(data, Int(round(max_obs / n_classes)))
         end
         (Symbol(replace(file, ".csv" => "")) => data)
     end
     data = Dict(data...)
     return data
 end
 
-function load_real_world(max_obs::Union{Nothing, Int}=nothing)
-    data_dir = joinpath(artifact"data","data/real_world")
+function load_real_world(max_obs::Union{Nothing,Int}=nothing; data_dir::Union{Nothing, String}=nothing)
+    if isnothing(data_dir)
+        data_dir = joinpath(artifact"data", "data/real_world")
+    end
     files = readdir(data_dir)
-    files = files[contains.(files,".csv")]
+    files = files[contains.(files, ".jls")]
     data = map(files) do file
-        df = CSV.read(joinpath(data_dir, file), DataFrame)
-        X = Matrix(df[:,Not(:target)])
-        X = permutedims(X)
-        y = convert(Matrix, df.target')
-        data = CounterfactualData(X,y)
+        counterfactual_data = Serialization.deserialize(joinpath(data_dir, file))
         if !isnothing(max_obs)
-            n_classes = length(unique(y))
-            data = undersample(data, Int(round(max_obs/n_classes)))
+            n_classes = length(unique(counterfactual_data.y))
+            counterfactual_data = undersample(counterfactual_data, Int(round(max_obs / n_classes)))
         end
-        (Symbol(replace(file, ".csv" => "")) => data)
+        (Symbol(replace(file, ".jls" => "")) => counterfactual_data)
     end
     data = Dict(data...)
     return data

src/data/utils.jl

@@ -31,7 +31,8 @@ function undersample(data::CounterfactualData, n_per_class::Int)
     classes_ = sort(unique(y_cls))
 
     idx = sort(reduce(vcat,[sample(findall(vec(y_cls.==cls)), n_per_class,replace=false) for cls in classes_]))
-    data = CounterfactualData(X[:,idx], y[:,idx])
+    data.X = X[:, idx]
+    data.y = y[:,idx]
 
     return data
 

src/experiments/functions.jl

@@ -170,6 +170,7 @@ function update_experiment!(experiment::Experiment, recourse_system::RecourseSys
 
     X′ = reduce(hcat, @.(selectdim(counterfactual(results), 3, indices_)))
     y′ = reduce(hcat, @.(selectdim(counterfactual_label(results), 1, indices_)))
+    println(y′)
 
     X[:, chosen_individuals] = X′
     y[:, chosen_individuals] = y′