work on data

Author: pat-alt

docs/src/paper/Project.toml

@@ -1,11 +1,14 @@
 [deps]
 AlgorithmicRecourseDynamics = "3d1ede72-abb8-4340-bf8e-2ae06849b5ec"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 CounterfactualExplanations = "2f13d31b-18db-44c1-bc43-ebaf2cff0be0"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0"
 LaplaceRedux = "c52c1a26-f7c5-402b-80be-ba1e638ad478"
 MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
+MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 RCall = "6f49c342-dc21-5d91-9882-a32aef131414"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"

…r/data_preprocessing/real_world_data.qmd …/data_preprocessing/_real_world_data.qmddocs/src/paper/data_preprocessing/real_world_data.qmd renamed to docs/src/paper/data_preprocessing/_real_world_data.qmd docs/src/paper/data_preprocessing/real_world_data.qmd renamed to docs/src/paper/data_preprocessing/_real_world_data.qmd

@@ -1,43 +1,39 @@
 ---
 title: Preprocessing Real-World Data
-jupyter: julia-1.7
 ---
 
 ```{julia}
-using Pkg; Pkg.activate("dev")
-```
+#| echo: false
 
-```{julia}
-include("dev/utils.jl")
-using AlgorithmicRecourseDynamics
-using CounterfactualExplanations, Flux, Plots, PlotThemes, Random, LaplaceRedux, LinearAlgebra
-theme(:wong)
+include("docs/src/paper/setup.jl")
+eval(setup)
 output_path = output_dir("real_world")
 www_path = www_dir("real_world")
 data_path = data_dir("real_world")
 ```
 
 ## California Housing Data
 
-Fetching the data using Python's `sklearn`:
+Fetching the data using Python's `sklearn` (run this in the Python REPL):
 
-```{python}
+```python
 from sklearn.datasets import fetch_california_housing
 df, y = fetch_california_housing(return_X_y=True, as_frame=True)
 df["target"] = y.values
-data_path = "../../artifacts/upload/data/real_world"
+data_path = "dev/artifacts/upload/data/real_world"
 import os
+if not os.path.isdir(os.path.join(data_path,"raw")):
+    os.makedirs(os.path.join(data_path,"raw"))
 df.to_csv(os.path.join(data_path,"raw/cal_housing.csv"), index=False)
 ```
 
 Loading the data into Julia session:
 
 ```{julia}
-using CSV, DataFrames, Statistics, StatsBase
 df = CSV.read(joinpath(data_path, "raw/cal_housing.csv"), DataFrame)
 # Features:
 X = Matrix(df[:,Not(:target)])
-dt = fit(ZScoreTransform, X, dims=1)
+dt = StatsBase.fit(ZScoreTransform, X, dims=1)
 StatsBase.transform!(dt, X)
 # Target:
 y = df.target

…er/data_preprocessing/synthetic_data.qmd …r/data_preprocessing/_synthetic_data.qmddocs/src/paper/data_preprocessing/synthetic_data.qmd renamed to docs/src/paper/data_preprocessing/_synthetic_data.qmd docs/src/paper/data_preprocessing/synthetic_data.qmd renamed to docs/src/paper/data_preprocessing/_synthetic_data.qmd

@@ -10,16 +10,19 @@ setup = quote
     using AlgorithmicRecourseDynamics.Models: model_evaluation
     using CounterfactualExplanations
     using CounterfactualExplanations: counterfactual, counterfactual_label
+    using CSV
     using DataFrames
     using Flux
     using Images
     using LaplaceRedux
     using Markdown
     using MLJBase
+    using MLUtils
     using Plots
     using Random
     using RCall
     using Serialization
+    using StatsBase
 
     # Setup
     Random.seed!(2023)              # global seed to allow for reproducibility