just missing the tables now

Author: pat-alt

_freeze/dev/notebooks/appendix/execute-results/html.json

@@ -6,6 +6,7 @@ CounterfactualExplanations = "2f13d31b-18db-44c1-bc43-ebaf2cff0be0"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 Gadfly = "c91e804a-d5a3-530f-b6f0-dfbca275c004"
+Gumbo = "708ec375-b3d6-5a57-a7ce-8257bf98657a"
 LaplaceRedux = "c52c1a26-f7c5-402b-80be-ba1e638ad478"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

build/dev/notebooks/appendix.html

@@ -8,6 +8,7 @@ format:
             echo: true
             eval: false
             warning: false
+        toc: true
 jupyter: julia-1.7
 ---
 

dev/Project.toml

@@ -9,10 +9,10 @@ using Pkg; Pkg.activate("dev")
 #| eval: true
 include("dev/utils.jl")
 using AlgorithmicRecourseDynamics
-using CounterfactualExplanations, Flux, Plots, PlotThemes, Random, LaplaceRedux, LinearAlgebra
+using CounterfactualExplanations, Flux, Plots, PlotThemes, Random, LaplaceRedux, LinearAlgebra, Images
 theme(:wong)
 output_path = output_dir("mitigation_strategies")
-www_path = www_dir("mitigation_strategies")
+www_path = www_dir("mitigation_strategies");
 ```
 
 ```{julia}
@@ -48,42 +48,6 @@ data_sets = filter(p -> p[1] in choices, catalogue)
 experiments = set_up_experiments(data_sets,models,generators)
 ```
 
-```{julia}
-using AlgorithmicRecourseDynamics.Models: model_evaluation
-plts = []
-for (exp_name, exp_) in experiments
-    for (M_name, M) in exp_.models
-        score = round(model_evaluation(M, exp_.test_data),digits=2)
-        plt = plot(M, exp_.test_data, title="$exp_name;\n $M_name ($score)")
-        # Errors:
-        ids = findall(vec(round.(probs(M, exp_.test_data.X)) .!= exp_.test_data.y))
-        x_wrongly_labelled = exp_.test_data.X[:,ids]
-        scatter!(plt, x_wrongly_labelled[1,:], x_wrongly_labelled[2,:], ms=7.5, color=:red, label="")
-        plts = vcat(plts..., plt)
-    end
-end
-plt = plot(plts..., layout=(length(choices),length(models)),size=(length(choices)*300,length(models)*300))
-savefig(plt, joinpath(www_path,"models_test_before.png"))
-```
-
-```{julia}
-using AlgorithmicRecourseDynamics.Models: model_evaluation
-plts = []
-for (exp_name, exp_) in experiments
-    for (M_name, M) in exp_.models
-        score = round(model_evaluation(M, exp_.train_data),digits=2)
-        plt = plot(M, exp_.train_data, title="$exp_name;\n $M_name ($score)")
-        # Errors:
-        ids = findall(vec(round.(probs(M, exp_.train_data.X)) .!= exp_.train_data.y))
-        x_wrongly_labelled = exp_.train_data.X[:,ids]
-        scatter!(plt, x_wrongly_labelled[1,:], x_wrongly_labelled[2,:], ms=7.5, color=:red, label="")
-        plts = vcat(plts..., plt)
-    end
-end
-plt = plot(plts..., layout=(length(choices),length(models)),size=(length(choices)*300,length(models)*300))
-savefig(plt, joinpath(www_path,"models_train_before.png"))
-```
-
 ```{julia}
 n_evals = 5
 n_rounds = 50
@@ -99,64 +63,7 @@ results = run_experiments(
 Serialization.serialize(joinpath(output_path,"results_synthetic.jls"),results)
 ```
 
-```{julia}
-using AlgorithmicRecourseDynamics.Models: model_evaluation
-plot_dict = Dict(key => Dict() for (key,val) in results)
-fold = 1
-for (name, res) in results
-    exp_ = res.experiment
-    plot_dict[name] = Dict(key => [] for (key,val) in exp_.generators)
-    rec_sys = exp_.recourse_systems[fold]
-    sys_ids = collect(exp_.system_identifiers)
-    M = length(rec_sys)
-    for m in 1:M
-        model_name, generator_name = sys_ids[m]
-        M = rec_sys[m].model
-        score = round(model_evaluation(M, exp_.test_data),digits=2)
-        plt = plot(M, exp_.test_data, title="$name;\n $model_name ($score)")
-        # Errors:
-        ids = findall(vec(round.(probs(M, exp_.test_data.X)) .!= exp_.test_data.y))
-        x_wrongly_labelled = exp_.test_data.X[:,ids]
-        scatter!(plt, x_wrongly_labelled[1,:], x_wrongly_labelled[2,:], ms=7.5, color=:red, label="")
-        plot_dict[name][generator_name] = vcat(plot_dict[name][generator_name], plt)
-    end
-end
-plot_dict = Dict(key => reduce(vcat, [plots[key] for plots in values(plot_dict)]) for (key, value) in generators)
-for (name, plts) in plot_dict
-    plt = plot(plts..., layout=(length(choices),length(models)),size=(length(choices)*300,length(models)*300))
-    savefig(plt, joinpath(www_path,"models_test_after_$(name).png"))
-end
-```
-
-```{julia}
-using AlgorithmicRecourseDynamics.Models: model_evaluation
-plot_dict = Dict(key => Dict() for (key,val) in results)
-fold = 1
-for (name, res) in results
-    exp_ = res.experiment
-    plot_dict[name] = Dict(key => [] for (key,val) in exp_.generators)
-    rec_sys = exp_.recourse_systems[fold]
-    sys_ids = collect(exp_.system_identifiers)
-    M = length(rec_sys)
-    for m in 1:M
-        model_name, generator_name = sys_ids[m]
-        M = rec_sys[m].model
-        data = rec_sys[m].data
-        score = round(model_evaluation(M, data),digits=2)
-        plt = plot(M, data, title="$name;\n $model_name ($score)")
-        # Errors:
-        ids = findall(vec(round.(probs(M, data.X)) .!= data.y))
-        x_wrongly_labelled = data.X[:,ids]
-        scatter!(plt, x_wrongly_labelled[1,:], x_wrongly_labelled[2,:], ms=7.5, color=:red, label="")
-        plot_dict[name][generator_name] = vcat(plot_dict[name][generator_name], plt)
-    end
-end
-plot_dict = Dict(key => reduce(vcat, [plots[key] for plots in values(plot_dict)]) for (key, value) in generators)
-for (name, plts) in plot_dict
-    plt = plot(plts..., layout=(length(choices),length(models)),size=(length(choices)*300,length(models)*300))
-    savefig(plt, joinpath(www_path,"models_train_after_$(name).png"))
-end
-```
+### Plots
 
 ```{julia}
 using Serialization
@@ -177,8 +84,58 @@ for (data_name, res) in results
 end
 ```
 
+#### Line Charts
+
+@fig-mit-line shows the evolution of the evaluation metrics over the course of the experiment.
+
+```{julia}
+#| eval: true
+#| fig-cap: "Line Charts"
+#| fig-subcap: 
+#|      - "California Housing"
+#|      - "Circles"
+#|      - "Credit Default"
+#|      - "GMSC"
+#|      - "Linearly Separable"
+#|      - "Moons"
+#|      - "Overlapping"
+#| layout-ncol: 1
+#| label: fig-mit-line
+img_files = readdir(www_path)[contains.(readdir(www_path),"line_chart") .&& .!contains.(readdir(www_path),"latent")]
+img_files = joinpath.(www_path,img_files)
+for img in img_files
+    display(load(img))
+end
+```
+
+#### Error Bar Charts
+
+@fig-mit-error shows the evaluation metrics at the end of the experiments.
+
+```{julia}
+#| eval: true
+#| fig-cap: "Error Bar Charts"
+#| fig-subcap: 
+#|      - "California Housing"
+#|      - "Circles"
+#|      - "Credit Default"
+#|      - "GMSC"
+#|      - "Linearly Separable"
+#|      - "Moons"
+#|      - "Overlapping"
+#| layout-ncol: 1
+#| label: fig-mit-error
+img_files = readdir(www_path)[contains.(readdir(www_path),"errorbar_chart") .&& .!contains.(readdir(www_path),"latent")]
+img_files = joinpath.(www_path,img_files)
+for img in img_files
+    display(load(img))
+end
+```
+
 ### Chart in paper
 
+@fig-mit-paper shows the chart that went into the paper.
+
 ```{julia}
 using DataFrames, Statistics
 df = results[:overlapping].output
@@ -234,6 +191,13 @@ img = Images.load(rcopy(R"temp_path"))
 Images.save(joinpath(www_path,"paper_synthetic_results.png"), img)
 ```
 
+```{julia}
+#| label: fig-mit-paper
+#| fig-cap: "Chart in paper"
+#| eval: true
+Images.load(joinpath(www_path,"paper_synthetic_results.png"))
+```
+
 
 ### Latent Space Search
 
@@ -284,8 +248,54 @@ for (data_name, res) in results
 end
 ```
 
+### Plots
+
+#### Line Charts
+
+@fig-mit-line-latent shows the evolution of the evaluation metrics over the course of the experiment.
+
+```{julia}
+#| eval: true
+#| fig-cap: "Line Charts"
+#| fig-subcap: 
+#|      - "Circles"
+#|      - "Linearly Separable"
+#|      - "Moons"
+#|      - "Overlapping"
+#| layout-ncol: 1
+#| label: fig-mit-line-latent
+img_files = readdir(www_path)[contains.(readdir(www_path),"line_chart") .&& contains.(readdir(www_path),"latent")]
+img_files = joinpath.(www_path,img_files)
+for img in img_files
+    display(load(img))
+end
+```
+
+#### Error Bar Charts
+
+@fig-mit-error-latent shows the evaluation metrics at the end of the experiments.
+
+```{julia}
+#| eval: true
+#| fig-cap: "Error Bar Charts"
+#| fig-subcap: 
+#|      - "Circles"
+#|      - "Linearly Separable"
+#|      - "Moons"
+#|      - "Overlapping"
+#| layout-ncol: 1
+#| label: fig-mit-error-latent
+img_files = readdir(www_path)[contains.(readdir(www_path),"errorbar_chart") .&& contains.(readdir(www_path),"latent")]
+img_files = joinpath.(www_path,img_files)
+for img in img_files
+    display(load(img))
+end
+```
+
 ### Chart in paper
 
+@fig-mit-latent-paper shows the chart that went into the paper.
+
 ```{julia}
 using DataFrames, Statistics
 df = results[:overlapping].output
@@ -341,6 +351,13 @@ img = Images.load(rcopy(R"temp_path"))
 Images.save(joinpath(www_path,"paper_synthetic_latent_results.png"), img)
 ```
 
+```{julia}
+#| label: fig-mit-latent-paper
+#| fig-cap: "Chart in paper"
+#| eval: true
+Images.load(joinpath(www_path,"paper_synthetic_latent_results.png"))
+```
+
 ## Real World
 
 ```{julia}
@@ -419,6 +436,8 @@ end
 
 ### Chart in paper
 
+@fig-mit-latent-paper shows the chart that went into the paper.
+
 ```{julia}
 using DataFrames, Statistics
 model_ = :FluxEnsemble
@@ -476,3 +495,10 @@ ggsave(temp_path,width=$ncol * $scale_,height=$nrow * $scale_ * 0.85)
 img = Images.load(rcopy(R"temp_path"))
 Images.save(joinpath(www_path,"paper_real_world_results.png"), img)
 ```
+
+```{julia}
+#| label: fig-mit-real-paper
+#| fig-cap: "Chart in paper"
+#| eval: true
+Images.load(joinpath(www_path,"paper_real_world_results.png"))
+```

dev/notebooks/appendix.qmd

@@ -221,7 +221,7 @@ end
 ```{julia}
 #| eval: true
 using Serialization
-results = Serialization.deserialize(joinpath(output_path,"results.jls"))
+results = Serialization.deserialize(joinpath(output_path,"results.jls"));
 ```
 
 ```{julia}
@@ -280,17 +280,42 @@ for img in img_files
 end
 ```
 
-### Tables
+### Bootstrap
 
-@tbl-results shows a summary of all results.
+```{julia}
+n_bootstrap = 1
+using AlgorithmicRecourseDynamics.Evaluation: evaluate_system
+using DataFrames
+df = DataFrame()
+for (key, val) in results
+    n_folds = length(val.experiment.recourse_systems)
+    for fold in 1:n_folds
+        for i in length(val.experiment.system_identifiers)
+            rec_sys = val.experiment.recourse_systems[fold][i]
+            model_name, gen_name = collect(val.experiment.system_identifiers)[i]
+            df_ = evaluate_system(rec_sys, val.experiment; n=n_bootstrap)
+            df_.model .= model_name
+            df_.generator .= gen_name
+            df_.fold .= fold
+            df = vcat(df, df_)
+        end
+    end
+end
+df = mapcols(x -> typeof(x) == Vector{Symbol} ? string.(x) : x, df)
+using RCall
+save_path = joinpath(output_path, "bootstrap.html")
+R"""
+dt <- DT::datatable($df) |>
+    DT::formatRound(columns=c("value"), digits=3)
+DT::saveWidget(dt, $save_path)
+"""
+```
 
 ```{julia}
-#| label: tbl-results
-#| tbl-cap: "Summary of all results"
 #| eval: true
-#| results: asis
-using AlgorithmicRecourseDynamics: kable
-kable(results, [50]; format="html")
+using Gumbo
+save_path = joinpath(output_path, "bootstrap.html")
+parsehtml(read(save_path, String))
 ```
 
 ### Chart in paper {#sec-app-synthetic-paper}