From c57dea767fea8a5fc54dfbe8a3cf7073d149c626 Mon Sep 17 00:00:00 2001 From: Ben Young Date: Mon, 11 Nov 2024 08:52:44 -0500 Subject: [PATCH 1/7] write model matrices and demand to JSON, #294 --- R/WriteModel.R | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/R/WriteModel.R b/R/WriteModel.R index 06fd7ce6..4eb1ac97 100644 --- a/R/WriteModel.R +++ b/R/WriteModel.R @@ -19,9 +19,23 @@ writeModelforAPI <-function(model, basedir){ writeSectorCrosswalk(model, dirs$data) } +#' Writes all model data and metadata components to JSON +#' @param model A complete EEIO model: a list with USEEIO model components and attributes. +#' @param basedir Base directory to write the model components to +#' @description Writes all model data and metadata components to JSON +#' @export +writeModeltoJSON <- function(model, basedir) { + dirs <- setWriteDirs(model,basedir) + prepareWriteDirs(model, dirs) + writeModelMatrices(model,"json",dirs$model) + writeModelDemandstoJSON(model,dirs$demands) + # writeModelMetadata(model,dirs) + # writeSectorCrosswalk(model, dirs$data) +} + #' Write model matrices as .csv or .bin files to output folder. #' @param model A complete EEIO model: a list with USEEIO model components and attributes. -#' @param to_format A string specifying the format of write-to file, can be "csv" or "bin". +#' @param to_format A string specifying the format of write-to file, can be "csv", "bin", or "json". #' @param outputfolder A directory to write matrices out to #' @description Writes model matrices as .csv or .bin files to output folder. #' @export @@ -46,6 +60,13 @@ writeModelMatrices <- function(model, to_format, outputfolder) { # Write x (Industry Output) or q (Commodity Output) to .bin files writeMatrixasBinFile(as.matrix(model$q), paste0(modelfolder, "/q.bin")) writeMatrixasBinFile(as.matrix(model$x), paste0(modelfolder, "/x.bin")) + } else if (to_format=="json") { + modelfolder <- paste0(outputfolder, "/matrix") + dir.create(modelfolder, showWarnings = FALSE) + for (matrix in c(matrix_ls, "q", "x")) { + writeMatrixtoJSON(as.matrix(model[[matrix]]), + paste0(modelfolder, "/", matrix, ".json")) + } } logging::loginfo(paste0("Model matrices written to ", modelfolder, ".")) } @@ -199,6 +220,15 @@ writeModelDemandstoJSON <- function(model, outputfolder) { logging::loginfo(paste0("Model demand vectors for API written to ", outputfolder, ".")) } +#' Write model matrix as JSON files to output folder. +#' @param model A complete EEIO model: a list with USEEIO model components and attributes. +#' @param outputpath A directory and file to write to. +#' @description Writes model objects. +writeMatrixtoJSON <- function(matrix, outputpath) { + mat <- jsonlite::toJSON(matrix, pretty = TRUE) + write(mat,outputpath) +} + #' Write model metadata (indicators and demands, sectors, and flows) as CSV files to output folder #' format for file is here https://github.com/USEPA/USEEIO_API/blob/master/doc/data_format.md #' @param model A complete EEIO model: a list with USEEIO model components and attributes. From 33d122d02dac7a17f6af5e2ef8322f6582ab68ec Mon Sep 17 00:00:00 2001 From: Ben Young Date: Mon, 11 Nov 2024 10:00:35 -0500 Subject: [PATCH 2/7] write model metadata to json, #294 --- R/WriteModel.R | 60 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/R/WriteModel.R b/R/WriteModel.R index 4eb1ac97..d3decd97 100644 --- a/R/WriteModel.R +++ b/R/WriteModel.R @@ -15,7 +15,7 @@ writeModelforAPI <-function(model, basedir){ prepareWriteDirs(model, dirs) writeModelMatrices(model,"bin",dirs$model) writeModelDemandstoJSON(model,dirs$demands) - writeModelMetadata(model,dirs) + writeModelMetadata(model,dirs,"csv") writeSectorCrosswalk(model, dirs$data) } @@ -29,7 +29,7 @@ writeModeltoJSON <- function(model, basedir) { prepareWriteDirs(model, dirs) writeModelMatrices(model,"json",dirs$model) writeModelDemandstoJSON(model,dirs$demands) - # writeModelMetadata(model,dirs) + writeModelMetadata(model,dirs,"json") # writeSectorCrosswalk(model, dirs$data) } @@ -104,7 +104,7 @@ writeModeltoXLSX <- function(model, outputfolder) { dirs <- setWriteDirs(model, file.path(rappdirs::user_data_dir(), "useeior", "Model_Builds", model$specs$Model)) prepareWriteDirs(model, dirs) - writeModelMetadata(model, dirs) + writeModelMetadata(model, dirs, "csv") metadata_tabs <- c("demands", "flows", "indicators", "sectors") if(is.null(model$SatelliteTables)){ metadata_tabs <- metadata_tabs[metadata_tabs != "flows"] @@ -225,16 +225,16 @@ writeModelDemandstoJSON <- function(model, outputfolder) { #' @param outputpath A directory and file to write to. #' @description Writes model objects. writeMatrixtoJSON <- function(matrix, outputpath) { - mat <- jsonlite::toJSON(matrix, pretty = TRUE) - write(mat,outputpath) + write(jsonlite::toJSON(matrix, pretty = TRUE), outputpath) } #' Write model metadata (indicators and demands, sectors, and flows) as CSV files to output folder #' format for file is here https://github.com/USEPA/USEEIO_API/blob/master/doc/data_format.md #' @param model A complete EEIO model: a list with USEEIO model components and attributes. #' @param dirs A named list of directories with model and data directory paths +#' @param to_format A string specifying the format of write-to file, can be "csv" or "json". #' @description Writes model metadata, including indicators and demands. -writeModelMetadata <- function(model, dirs) { +writeModelMetadata <- function(model, dirs, to_format="csv") { # Load metadata fields for API fields <- configr::read.config(system.file("extdata/USEEIO_API_fields.yml", package="useeior")) @@ -268,7 +268,11 @@ writeModelMetadata <- function(model, dirs) { df <- rbind(df, cbind.data.frame(model_fields)) } } - utils::write.csv(df, model_desc, na = "", row.names = FALSE, fileEncoding = "UTF-8") + if(to_format == "csv") { + utils::write.csv(df, model_desc, na = "", row.names = FALSE, fileEncoding = "UTF-8") + } else if(to_format == "json") { + write(jsonlite::toJSON(df, pretty = TRUE), file.path(dirs$data, "models.json")) + } if(!is.null(model$Indicators)) { # Write indicators to indicators.csv @@ -280,16 +284,24 @@ writeModelMetadata <- function(model, dirs) { indicators <- indicators[,fields$indicators] checkNamesandOrdering(indicators$Name, rownames(model$C), "code in indicators.csv and rows in C matrix") - utils::write.csv(indicators, paste0(dirs$model, "/indicators.csv"), na = "", - row.names = FALSE, fileEncoding = "UTF-8") + if(to_format == "csv") { + utils::write.csv(indicators, paste0(dirs$model, "/indicators.csv"), na = "", + row.names = FALSE, fileEncoding = "UTF-8") + } else if(to_format == "json") { + write(jsonlite::toJSON(indicators, pretty = TRUE), file.path(dirs$model, "indicators.json")) + } } # Write demands to demands.csv demands <- model$DemandVectors$meta demands$Index <- c(1:nrow(demands)-1) demands <- demands[,fields$demands] - utils::write.csv(demands, paste0(dirs$model, "/demands.csv"), na = "", - row.names = FALSE, fileEncoding = "UTF-8") + if(to_format == "csv") { + utils::write.csv(demands, paste0(dirs$model, "/demands.csv"), na = "", + row.names = FALSE, fileEncoding = "UTF-8") + } else if(to_format == "json") { + write(jsonlite::toJSON(demands, pretty = TRUE), file.path(dirs$model, "demands.json")) + } # Write sectors to csv sectors <- model[[gsub("y", "ies", model$specs$CommodityorIndustryType)]] @@ -300,8 +312,13 @@ writeModelMetadata <- function(model, dirs) { sectors <- sectors[, fields$sectors] checkNamesandOrdering(sectors$ID, rownames(model$L), "code in sectors.csv and rows in L matrix") - utils::write.csv(sectors, paste0(dirs$model, "/sectors.csv"), na = "", - row.names = FALSE, fileEncoding = "UTF-8") + if(to_format == "csv") { + utils::write.csv(sectors, paste0(dirs$model, "/sectors.csv"), na = "", + row.names = FALSE, fileEncoding = "UTF-8") + } else if(to_format == "json") { + sectors$Description <- NULL + write(jsonlite::toJSON(sectors, pretty = TRUE), file.path(dirs$model, "sectors.json")) + } if(!is.null(model$SatelliteTables)) { # Write flows to csv @@ -313,8 +330,12 @@ writeModelMetadata <- function(model, dirs) { flows <- flows[, fields$flows] #checkNamesandOrdering(flows$ID, rownames(model$B), # "flows in flows.csv and rows in B matrix") - utils::write.csv(flows, paste0(dirs$model, "/flows.csv"), na = "", - row.names = FALSE, fileEncoding = "UTF-8") + if(to_format == "csv") { + utils::write.csv(flows, paste0(dirs$model, "/flows.csv"), na = "", + row.names = FALSE, fileEncoding = "UTF-8") + } else if(to_format == "json") { + write(jsonlite::toJSON(flows, pretty = TRUE), file.path(dirs$model, "flows.json")) + } } # Write years to csv @@ -328,8 +349,13 @@ writeModelMetadata <- function(model, dirs) { } checkNamesandOrdering(years$ID, colnames(model$Rho), "years in years.csv and cols in Rho matrix") - utils::write.csv(years, paste0(dirs$model, "/years.csv"), na = "", - row.names = FALSE, fileEncoding = "UTF-8") + if(to_format == "csv") { + utils::write.csv(years, paste0(dirs$model, "/years.csv"), na = "", + row.names = FALSE, fileEncoding = "UTF-8") + } else if(to_format == "json") { + #todo: is this needed for json? + write(jsonlite::toJSON(years, pretty = TRUE), file.path(dirs$model, "years.json")) + } # Write session info to R sessioninfo.txt inside the model folder writeSessionInfotoFile(dirs$model) From 948e39bf6f9b626fd7127104c05676776b8b9639 Mon Sep 17 00:00:00 2001 From: Ben Young Date: Mon, 11 Nov 2024 10:09:45 -0500 Subject: [PATCH 3/7] write sector cw to json, #294 --- R/WriteModel.R | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/R/WriteModel.R b/R/WriteModel.R index d3decd97..001734a0 100644 --- a/R/WriteModel.R +++ b/R/WriteModel.R @@ -16,7 +16,7 @@ writeModelforAPI <-function(model, basedir){ writeModelMatrices(model,"bin",dirs$model) writeModelDemandstoJSON(model,dirs$demands) writeModelMetadata(model,dirs,"csv") - writeSectorCrosswalk(model, dirs$data) + writeSectorCrosswalk(model,dirs$data,"csv") } #' Writes all model data and metadata components to JSON @@ -30,7 +30,7 @@ writeModeltoJSON <- function(model, basedir) { writeModelMatrices(model,"json",dirs$model) writeModelDemandstoJSON(model,dirs$demands) writeModelMetadata(model,dirs,"json") - # writeSectorCrosswalk(model, dirs$data) + writeSectorCrosswalk(model,dirs$data,"json") } #' Write model matrices as .csv or .bin files to output folder. @@ -365,14 +365,19 @@ writeModelMetadata <- function(model, dirs, to_format="csv") { #' Write the model sector crosswalk as .csv file #' @param model A complete EEIO model: a list with USEEIO model components and attributes. #' @param outputfolder A directory to write model sector crosswalk -#' @description Writes the model sector crosswalk as .csv file -writeSectorCrosswalk <- function(model, outputfolder){ +#' @param to_format A string specifying the format of write-to file, can be "csv" or "json". +#' @description Writes the model sector crosswalk as .csv or .json file +writeSectorCrosswalk <- function(model, outputfolder, to_format="csv"){ crosswalk <- prepareModelSectorCrosswalk(model) crosswalk$ModelSchema <- "" - utils::write.csv(crosswalk, paste0(outputfolder, "/sectorcrosswalk.csv"), - na = "", row.names = FALSE, fileEncoding = "UTF-8") - logging::loginfo(paste0("Sector crosswalk written as sectorcrosswalk.csv to ", - outputfolder, ".")) + if(to_format == "csv") { + utils::write.csv(crosswalk, paste0(outputfolder, "/sectorcrosswalk.csv"), + na = "", row.names = FALSE, fileEncoding = "UTF-8") + } else if (to_format == "json") { + write(jsonlite::toJSON(crosswalk, pretty = TRUE), + file.path(outputfolder, "sectorcrosswalk.json")) + } + logging::loginfo(paste0("Sector crosswalk written to ", outputfolder, ".")) } #' Write out session information to a "Rsessioninfo.txt file in the given path From 22f3196730e1d3974e25553f57595519fa8d7dbb Mon Sep 17 00:00:00 2001 From: Ben Young Date: Mon, 11 Nov 2024 10:11:22 -0500 Subject: [PATCH 4/7] update documentation --- NAMESPACE | 1 + man/writeMatrixtoJSON.Rd | 16 ++++++++++++++++ man/writeModelMatrices.Rd | 2 +- man/writeModelMetadata.Rd | 4 +++- man/writeModeltoJSON.Rd | 16 ++++++++++++++++ man/writeSectorCrosswalk.Rd | 6 ++++-- 6 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 man/writeMatrixtoJSON.Rd create mode 100644 man/writeModeltoJSON.Rd diff --git a/NAMESPACE b/NAMESPACE index c504c06d..9432727e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -26,5 +26,6 @@ export(testCalculationFunctions) export(testVisualizationFunctions) export(writeModelMatrices) export(writeModelforAPI) +export(writeModeltoJSON) export(writeModeltoXLSX) import(ggplot2) diff --git a/man/writeMatrixtoJSON.Rd b/man/writeMatrixtoJSON.Rd new file mode 100644 index 00000000..34233706 --- /dev/null +++ b/man/writeMatrixtoJSON.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/WriteModel.R +\name{writeMatrixtoJSON} +\alias{writeMatrixtoJSON} +\title{Write model matrix as JSON files to output folder.} +\usage{ +writeMatrixtoJSON(matrix, outputpath) +} +\arguments{ +\item{outputpath}{A directory and file to write to.} + +\item{model}{A complete EEIO model: a list with USEEIO model components and attributes.} +} +\description{ +Writes model objects. +} diff --git a/man/writeModelMatrices.Rd b/man/writeModelMatrices.Rd index e053367a..db053a16 100644 --- a/man/writeModelMatrices.Rd +++ b/man/writeModelMatrices.Rd @@ -9,7 +9,7 @@ writeModelMatrices(model, to_format, outputfolder) \arguments{ \item{model}{A complete EEIO model: a list with USEEIO model components and attributes.} -\item{to_format}{A string specifying the format of write-to file, can be "csv" or "bin".} +\item{to_format}{A string specifying the format of write-to file, can be "csv", "bin", or "json".} \item{outputfolder}{A directory to write matrices out to} } diff --git a/man/writeModelMetadata.Rd b/man/writeModelMetadata.Rd index fc1ba983..473a9908 100644 --- a/man/writeModelMetadata.Rd +++ b/man/writeModelMetadata.Rd @@ -5,12 +5,14 @@ \title{Write model metadata (indicators and demands, sectors, and flows) as CSV files to output folder format for file is here https://github.com/USEPA/USEEIO_API/blob/master/doc/data_format.md} \usage{ -writeModelMetadata(model, dirs) +writeModelMetadata(model, dirs, to_format = "csv") } \arguments{ \item{model}{A complete EEIO model: a list with USEEIO model components and attributes.} \item{dirs}{A named list of directories with model and data directory paths} + +\item{to_format}{A string specifying the format of write-to file, can be "csv" or "json".} } \description{ Writes model metadata, including indicators and demands. diff --git a/man/writeModeltoJSON.Rd b/man/writeModeltoJSON.Rd new file mode 100644 index 00000000..82787a80 --- /dev/null +++ b/man/writeModeltoJSON.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/WriteModel.R +\name{writeModeltoJSON} +\alias{writeModeltoJSON} +\title{Writes all model data and metadata components to JSON} +\usage{ +writeModeltoJSON(model, basedir) +} +\arguments{ +\item{model}{A complete EEIO model: a list with USEEIO model components and attributes.} + +\item{basedir}{Base directory to write the model components to} +} +\description{ +Writes all model data and metadata components to JSON +} diff --git a/man/writeSectorCrosswalk.Rd b/man/writeSectorCrosswalk.Rd index 2416907d..75ec8ec6 100644 --- a/man/writeSectorCrosswalk.Rd +++ b/man/writeSectorCrosswalk.Rd @@ -4,13 +4,15 @@ \alias{writeSectorCrosswalk} \title{Write the model sector crosswalk as .csv file} \usage{ -writeSectorCrosswalk(model, outputfolder) +writeSectorCrosswalk(model, outputfolder, to_format = "csv") } \arguments{ \item{model}{A complete EEIO model: a list with USEEIO model components and attributes.} \item{outputfolder}{A directory to write model sector crosswalk} + +\item{to_format}{A string specifying the format of write-to file, can be "csv" or "json".} } \description{ -Writes the model sector crosswalk as .csv file +Writes the model sector crosswalk as .csv or .json file } From 03df863477a95ea23920b99c32f26b0f15c7434b Mon Sep 17 00:00:00 2001 From: Ben Young Date: Mon, 11 Nov 2024 10:15:36 -0500 Subject: [PATCH 5/7] include writing to JSON in tests --- tests/test_model_build.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_model_build.R b/tests/test_model_build.R index 8f36d433..2b19dd14 100644 --- a/tests/test_model_build.R +++ b/tests/test_model_build.R @@ -129,6 +129,7 @@ writeModeltoXLSX(model, ".") ## USEEIOv2.3 Summary, commodity model with GHGs and Import Factors m <- "USEEIOv2.3-s-GHG-19" model <- buildModel(m) +writeModeltoJSON(model, ".") printValidationResults(model) testCalculationFunctions(model) testVisualizationFunctions(model) @@ -137,6 +138,7 @@ testVisualizationFunctions(model) m <- "GAEEIOv1.0-GHG-19" model <- buildModel(m) printValidationResults(model) +writeModeltoJSON(model, ".") writeModeltoXLSX(model, ".") testCalculationFunctions(model) testVisualizationFunctions(model) From 8e5c90875884598fc97d870077f20b020d0c4690 Mon Sep 17 00:00:00 2001 From: Ben Young Date: Mon, 11 Nov 2024 10:22:33 -0500 Subject: [PATCH 6/7] fix documentation issue --- R/WriteModel.R | 2 +- man/writeMatrixtoJSON.Rd | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R/WriteModel.R b/R/WriteModel.R index 001734a0..31c625ff 100644 --- a/R/WriteModel.R +++ b/R/WriteModel.R @@ -221,7 +221,7 @@ writeModelDemandstoJSON <- function(model, outputfolder) { } #' Write model matrix as JSON files to output folder. -#' @param model A complete EEIO model: a list with USEEIO model components and attributes. +#' @param matrix A matrix to be written #' @param outputpath A directory and file to write to. #' @description Writes model objects. writeMatrixtoJSON <- function(matrix, outputpath) { diff --git a/man/writeMatrixtoJSON.Rd b/man/writeMatrixtoJSON.Rd index 34233706..3a9e5829 100644 --- a/man/writeMatrixtoJSON.Rd +++ b/man/writeMatrixtoJSON.Rd @@ -7,9 +7,9 @@ writeMatrixtoJSON(matrix, outputpath) } \arguments{ -\item{outputpath}{A directory and file to write to.} +\item{matrix}{A matrix to be written} -\item{model}{A complete EEIO model: a list with USEEIO model components and attributes.} +\item{outputpath}{A directory and file to write to.} } \description{ Writes model objects. From c8a564764ef9076ab2d08866705a3c9d823e77e5 Mon Sep 17 00:00:00 2001 From: Ben Young Date: Mon, 11 Nov 2024 10:34:41 -0500 Subject: [PATCH 7/7] don't overwrite json for models.json --- R/WriteModel.R | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/R/WriteModel.R b/R/WriteModel.R index 31c625ff..2705dda7 100644 --- a/R/WriteModel.R +++ b/R/WriteModel.R @@ -240,7 +240,7 @@ writeModelMetadata <- function(model, dirs, to_format="csv") { package="useeior")) # Write model description to models.csv - model_desc <- file.path(dirs$data, "models.csv") + model_desc <- file.path(dirs$data, paste0("models.", to_format)) ID <- model$specs$Model Name <- model$specs$Model Location <- model$specs$ModelRegionAcronyms[1] @@ -262,8 +262,12 @@ writeModelMetadata <- function(model, dirs, to_format="csv") { if (!file.exists(model_desc)) { df <- cbind.data.frame(model_fields) } else { - df <- utils::read.table(model_desc, sep = ",", header = TRUE, - stringsAsFactors = FALSE, check.names = FALSE) + if(to_format == "csv") { + df <- utils::read.table(model_desc, sep = ",", header = TRUE, + stringsAsFactors = FALSE, check.names = FALSE) + } else if (to_format == "json") { + df <- jsonlite::fromJSON(model_desc) + } if (!ID%in%df$ID) { df <- rbind(df, cbind.data.frame(model_fields)) }