diff --git a/.gitignore b/.gitignore index 786ad64..9e44683 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .vscode .DS_Store .ipynb_checkpoints +.virtual_documents *.html __pycache__ @@ -15,5 +16,9 @@ up_template.egg-info/* dockers/*/Dockerfile .env +# Ignore data and outputs on repo +data +output + # Keep the HTML resources with the repo !resources/*.html \ No newline at end of file diff --git a/README.md b/README.md index ed58462..67396a7 100644 --- a/README.md +++ b/README.md @@ -7,21 +7,56 @@ The foundation of all tools in this package is the [NetlOlca](https://doi.org/10 ## Jupyter Notebook Unit Process Template -One of the main use cases for this tool is a new template based on Jupyter Notebook for reporting LCA unit processes. +The unit process template is a markdown file, organized by headers, to capture the essential information that goes into the creation of a life cycle unit process. +Capturing key metadata is critical for reproducibility of unit processes and for transparency to enable its application. -The new template is provided at the top-level of this repository (up_template.ipynb). +Two Python classes were developed to support the generation of this report: -![](img/package_uml.png) +1. NetlOlcaReport (up_template/NetlOlcaReport.py) +2. Interface (up_template/Interface.py) -Two additional classes were developed to support this interactive report generator: +The NetlOlcaReport class provides the methods for converting openLCA entity data into usable data frames and summaries and writing them into markdown format that can be converted into other reportable formats (e.g., HTML, Microsoft Word, and PDF). +The file format conversion depends on a user's local installation of [pandoc](https://pandoc.org/), a free and open-source tool for converting between different markup formats. -1. NetlOlcaReport (netlolca/NetlOlcaReport.py) -2. Interface (netlolca/Interface.py) +To create a blank template in markdown: -The NetlOlcaReport class provides intermediate methods for converting openLCA entity data into usable data frames and generating a summary of the results into a markdown format that can be converted into other reportable formats (e.g., HTML, Microsoft Word, and PDF). -The file format conversion depends on a user's local installation of [pandoc](https://pandoc.org/), a free and open-source tool for converting between different markup formats. +```python +>>> from netlolca import NetlOlca +>>> from up_template import NetlOlcaReport +>>> r = NetlOlcaReport(NetlOlca()) +>>> r.save_markdown(blank=True) +``` + +You may edit this markdown file before rendering to HTML. +If you rename the file, be sure to update the reference name in the Python class. + +```python +>>> r.reference_name = "new_report" # don't include the file extension +``` + +To generate the HTML version of the report, run the conversion method (PDF and Microsoft Word formats are also available). + +```python +>>> r.convert_to_html() # requires pandoc install +>>> r.convert_to_pdf() # requires additional LaTeX install +>>> r.convert_to_word() +``` + +Automated methods are available in the NetlOlcaReport class, which attempt to read and extract metadata from an openLCA database and map it to the report template. +Similar to above, the steps to connect the NetlOlcaReport class to an openLCA database are as follows: + +```python +>>> n = NetlOlca() +>>> n.connect() # establish connection via IPC service (default port 8080) +>>> n.read() # read database UUIDs +>>> r = NetlOlcaReport(n) +>>> ps_uuid = r.product_systems[0] # Get UUID of lone Product System +>>> r.fetch_data(ps_uuid) # Scrape database for metadata +>>> r.save_to_html() # create MD and HTML report files +``` -The recommended app for running the UP Template is [JupyterLab](https://jupyter.org/), the latest web-based interactive development environment for computational notebooks. +The Interface.py is an experimental class that guides users through metadata review and gap-filling. +This is best accommodated through [JupyterLab](https://jupyter.org/), the latest web-based interactive development environment for computational notebooks. JupyterLab may be installed using Python's `pip` or conda's `install` commands. To start Jupyter Lab, run the following command (after installing) in the parent folder where your up_template.ipynb is located: @@ -37,11 +72,12 @@ This should start the Jupyter notebook server and automatically launch the landi up-template/ ├── calculations/ <- store for auxiliary Excel workbooks to document │ │ calculations in the UP template - │ └── calculation_template.xlsx <-- template for calculations + │ └── calculation_template.xlsx <- template for calculations │ - ├── data/ <- empty folder for data files (e.g., JSON-LD) + ├── (data/) <- folder for data files (e.g., JSON-LD) │ ├── dockers/ (for getting things to run in Docker) + │ ├── USERGUIDE.md <- Docker instructions │ ├── gdt_server/ │ │ ├── README.md │ │ ├── build.bat @@ -56,6 +92,7 @@ This should start the Jupyter notebook server and automatically launch the landi │ ├── build.sh │ ├── get-docker.bat │ ├── get-docker.sh + │ ├── requirements.txt │ ├── run.bat │ └── run.sh │ @@ -65,13 +102,7 @@ This should start the Jupyter notebook server and automatically launch the landi │ └── Makefile <- source files to documentation (e.g., html) │ ├── img/ <- image resources for notebooks and README - │ ├── banner.png (293 KB) - │ ├── boundary_diagram.png (205 KB) - │ ├── logo_doe-netl_white_1000x176.png (30 KB) - │ ├── netl_logo_100x52.png (14 KB) - │ ├── netl_logo_1153x599.png (26 KB) - │ ├── netl_logo2.svg (2 KB) - │ ├── netl_logo2_320x251.png (13 KB) + │ ├── ipynb-token.png (92 KB) │ └── package_uml.png (281 KB) │ ├── (output/) <- Created when running UP Template to store the @@ -79,11 +110,13 @@ This should start the Jupyter notebook server and automatically launch the landi │ ├── resources/ │ ├── after_body.html + │ ├── banner.png <- header in up_template.ipynb │ ├── before_body.html - │ ├── banner.png - │ ├── netl_logo_100x52.png - │ ├── logo_doe-netl_white_1000x176.png - │ ├── (other logo files?) + │ ├── boundary_diagram.png + │ ├── logo_doe-netl_white_1000x176.png <- footer in template.docx + │ ├── netl_logo_100x52.png <- header in template.docx + │ ├── netl_logo_1153x599.png <- header in before_body.html + │ ├── styles.css │ └── template.docx │ ├── up_template/ <- Source code for this package. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9844abe --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "up_template" +version = "4.5.0" +description = "The NETL Unit Process Template and Report Generator" +readme = "README.md" +authors = [ + { name = "Tyler W. Davis" }, + { name = "Priyadarshini" }, + { name = "Joseph Chou" }, + { name = "Matt Jamieson", email = "matthew.jamieson@netl.doe.gov" } +] +license = "CC0-1.0" +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "netlolca @ git+https://github.com/NETL-RIC/netlolca", + "openpyxl", + "pandas", + "sympy", + "tabulate", + "jupyterlab", +] + +[project.urls] +Homepage = "https://github.com/NETL-RIC/up_template" + +[tool.setuptools] +packages = ["up_template"] diff --git a/resources/before_body.html b/resources/before_body.html index bfa5b78..7bbba8b 100644 --- a/resources/before_body.html +++ b/resources/before_body.html @@ -6,8 +6,9 @@

NETL Life Cycle Inventory Data Process Document File

diff --git a/resources/styles.css b/resources/styles.css new file mode 100644 index 0000000..349408b --- /dev/null +++ b/resources/styles.css @@ -0,0 +1,28 @@ +.table-wrapper { + overflow-x: scroll; + max-height:900px; +} + +.table-wrapper > table > thead > tr { + position: sticky; + top: 0; + background-color: white; +} + + +.horizontal-scroll-except-first-column { + overflow-x: auto; + max-height:900px; +} + +.horizontal-scroll-except-first-column > table > * > tr > th:first-child, +.horizontal-scroll-except-first-column > table > * > tr > td:first-child { + position: sticky; + left: 0; + z-index: 2; + background-color: white; +} + +dt { + font-weight: bold; +} \ No newline at end of file diff --git a/setup.py b/setup.py index 79fbdf5..6a7711f 100644 --- a/setup.py +++ b/setup.py @@ -19,11 +19,12 @@ setup( name="up_template", - version="3.0.0", + version="4.5.0", license="CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", packages=['up_template'], install_requires=[ "netlolca @ git+https://github.com/NETL-RIC/netlolca#egg=netlolca", + "openpyxl", "pandas", "sympy", "tabulate", @@ -36,7 +37,7 @@ long_description_content_type="text/markdown", url="https://github.com/NETL-RIC/up_template", classifiers=[ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication', 'Programming Language :: Python :: 3', diff --git a/up_template.ipynb b/up_template.ipynb index 32e21a4..588ee0f 100644 --- a/up_template.ipynb +++ b/up_template.ipynb @@ -292,7 +292,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.10" + "version": "3.13.5" } }, "nbformat": 4, diff --git a/up_template/NetlOlcaReport.py b/up_template/NetlOlcaReport.py index 8eb1dc5..8cd17ad 100644 --- a/up_template/NetlOlcaReport.py +++ b/up_template/NetlOlcaReport.py @@ -12,8 +12,20 @@ This module is a key part of the NETL Life Cycle Analysis (LCA) toolkit, designed for streamlined analysis and report generation. +Examples: + +>>> from up_template import NetlOlcaReport +>>> from netlolca import NetlOlca +>>> n = NetlOlca() # initialize data manager +>>> n.connect() # establish with IPC Server +>>> n.read() # read openLCA database +>>> r = NetlOlcaReport(n) # creates 'output' folder +>>> ps_id = r.product_systems[0] # find 1st product system UUID +>>> r.fetch_data(ps_id) # automated data scraping +>>> r.convert_to_html() # create .md; render to .html + Last Edited: - 2024-11-01 + 2026-03-13 """ __all__ = [ "NetlOlcaReport", @@ -52,7 +64,7 @@ ############################################################################## # CLASSES ############################################################################## -class NetlOlcaReport: +class NetlOlcaReport(object): """A report generating class for the Jupyter Notebook unit process template. @@ -174,6 +186,16 @@ def __init__(self, netlolca_instance): "Government or any agency thereof." ) + # \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ + # Class Properties + # //////////////////////////////////////////////////////////////////////// + @property + def product_systems(self): + """List of product system UUIDs""" + return self.netlolca.get_spec_ids( + self.netlolca.get_spec_class("Product system") + ) + # \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ # Class Function Definitions # //////////////////////////////////////////////////////////////////////// @@ -223,6 +245,11 @@ def convert(self, from_file, to_file): '--section-divs', '--css', 'https://cdnjs.cloudflare.com/ajax/libs/concrete.css/3.0.0/concrete.min.css', + '--css', + 'resources/styles.css', + '--metadata', + f'pagetitle={self.reference_name}', + '--title-prefix=NETL UP Library', '--mathjax', '--include-before-body', 'resources/before_body.html', @@ -314,6 +341,9 @@ def create_report_markdown(self): Notes ----- + This is a semi-automated report for the natural gas combustion unit + process. + Class attributes are updated with fetched data, see :func:`fetch_data`. Returns @@ -340,10 +370,22 @@ def create_report_markdown(self): if len(self.sources) > 0: sources_md = "\n\n".join(self.sources) + # Issue #8. Scenario attribute table (from workbook in DATA_DIR). + try: + scenario_table_md = scenario_attribute_table( + workbook_filename="DS_Stage5_O_Natural_Gas_Combustion_2014.01.xlsx", + sheet_name="PS", + header_row_excel=6, + start_col_excel="D", + ) + except Exception as e: + logging.warning(f"Scenario attribute table could not be created: {e}") + scenario_table_md = "No scenario attributes available." + # Add the calculations content from the filled workbook calculations_md = self.calculations_content or "No calculations available." - # Create markdown report + # Issue #8. Create markdown report (this has hard-coded values). report_md = f"""# Overview ## Process Name @@ -353,81 +395,141 @@ def create_report_markdown(self): {ref_flow} ## Brief Description -{process_desc} - -# Metadata {process_doc_md} -## Relevant Flows Included: -Releases to Air -: - [ ] Greenhouse Gases - - [ ] Criteria Air Pollutants - - [ ] Other +# Process Description -Releases to Water -: - [ ] Inorganic Emissions - - [ ] Organic Emissions - - [ ] Other +## Goal & Scope +{process_desc} -Releases to Soil -: - [ ] Inorganic Emissions - - [ ] Organic Emissions - - [ ] Other +## Process Boundary +{boundary_desc} -Water Usage -: - [ ] Water Demand - - [ ] Water Consumption +![](data/diagram.png) +### Input Flows -# Process Description +The table below provides a list of resource flows for this process. -## Goal & Scope -{project_doc_md} +
-## Boundary & Description -{boundary_desc} +{input_flows_md} -## Methods +
-### Block Flow Diagram -Link to your block flow diagram. -For example: +### Output Flows -```sh -![](data/diagram.png) -``` +The table below provides a list of emission flows for this process. -### Input Flows -{input_flows_md} +
-## Output Flows {output_flows_md} -### Process Parameters +
+ +## Preferred Providers + +_For each input product flow, add a statement as to its potential provider process(es)._ + +## Suggested Downstream Uses + +_Based on the quantitative reference flow, add a statement as to what processes may use this flow as a resource (if any)._ + + +# Methods + +## Scenarios + +The following table describes the scenarios used in this process. + +{scenario_table_md} + +## Process Parameters + +The following table provides parameter quantities for each of the scenarios defined [above](#scenarios). +These parameters represent the emissions found in the [output flow table](#output-flows). + {param_table_md} -### Allocation +## Allocation {allocation_md} -### Calculations -{calculations_md} +## Calculations and Supplements + +This report utilized the following supplemental files: + +- .xlsx + * _Include any workbooks used._ +- .py + * _Include any Python scripts (or Jupyter notebooks) used._ +- .zolca + * _Include the openLCA .zolca (or JSON-LD)._ + + +## Modeling Assumptions + +- _List the modeling assumptions, which may be found in "Modeling constants" section for process documentation in openLCA._ + +# Data + +## Data Quality + +The following Data Quality Indicator (DQI) scores are assigned to this process +using the DQI Matrix. + +
+ +| DQI Indicator | Score | Basis for Score | +|--------------|:-----:|----------------| +| Source Reliability | **x** | _description_ | +| Completeness | **x** | _description_ | +| Temporal Correlation | **x** | _description_ | +| Geographical Correlation | **x** | _description_ | +| Technological Correlation | **x** | _description_ | + +
+ +## Data Source Information + +_Copy the values from the named fields below from openLCA's process documentation into the table._ + +| Element | Description | +|--------|-------------| +| Data Completeness | | +| Data Selection | | +| Data Treatment | | +| Sampling Procedure | | +| Data Collection Period | | ## References {sources_md} + +# Use Information + +Use Advice +: _Copy text from "Intended Application" field in openLCA._ + +Access and Restrictions +: _Copy text from "Access and use restrictions" field in openLCA._ + + # Document Control Information -Date Created +Report Created: : {create_date} -Point of Contact +Point of Contact: : {project_poc_md} -Revision History +Revision History: : {version_number} -How to Cite This Document -: TBA +How to Cite This Document: +: _Copied from EDX submission._ + +Archived +: False # Disclaimer/Terms of Use {self.disclaimer} @@ -445,75 +547,158 @@ def empty_template(self): """ empty_rp = f"""# Overview +_The following template has instructions included under each section in italics._ + ## Process Name +_Process naming format should be based on the following: "Base name; treatment received, production route(s), standard(s) fulfilled; production or consumption type, location type; quantitative flow properties"_ + ## Reference Flow +_Copy and paste the quantitative reference flow for the process including the flow amount, units, and name._ + ## Brief Description -# Metadata +_High-level process summary that communicates the basic technology and what is included in the boundary; see example below._ -## Relevant Flows Included: +- Process Type: _State whether this is a Unit Process or System Process_ +- Inventory Method: _State whether this is Attributional or Consequential_ +- Technology Type: _High level technology categorization, such as natural gas, or bio fuels_ +- Unit Process Boundary: _State whether the process is Cradle-to-Grave, Cradle-to-Gate, Gate-to-Gate, etc._ +- Co-products: _List names, if applicable; otherwise, N/A_ +- Co-product allocation: _If applicable; otherwise, N/A. Examples include system expansion and mass allocation; list what is used and if there are any restrictions._ +- Scenarios: _Yes or no (e.g., link to [Scenarios](#scenarios) section)_ +- Location: _Geographic region associated with the unit process_ +- Valid From: _Year or date format_ +- Valid Until: _Year or date format_ +- Unit Process Creation Date: _May be different from report date_ +- Data Quantity Score: _The process DQI score (e.g., link to [Data Quality](#data-quality) for explanation of scoring)_ +- Data Completeness: _Link to [Data Source Information](#data-source-information) for more information_ -Releases to Air -: - [ ] Greenhouse Gases - - [ ] Criteria Air Pollutants - - [ ] Other +_Conclude with the stated intended use._ -Releases to Water -: - [ ] Inorganic Emissions - - [ ] Organic Emissions - - [ ] Other +# Process Description -Releases to Soil -: - [ ] Inorganic Emissions - - [ ] Organic Emissions - - [ ] Other +## Goal & Scope -Water Usage -: - [ ] Water Demand - - [ ] Water Consumption +_Define the purpose, and boundaries of the unit process, specifying what is included or excluded in the inventory._ +## Process Boundary -# Process Description +_Describes the specific cutoff points that determine which inputs, outputs, and life cycle stages are included or excluded in the inventory._ -## Goal & Scope - +_Also, include a graphic of the box diagram for your process._ -## Boundary & Description - +## Input Flows -## Methods +_Provide an introduction (e.g., "The table below provides a list of resource flows for this process.") and provide a table of resources with columns: 'Compartment', 'Flow Name', 'Quantity', 'Unit', and 'DQI' (automated in Python)._ -### Block Flow Diagram +## Output Flows -### Input Flows +_Provide an introduction (e.g., "The table below provides a list of emission flows for this process.") and provide a table of resources with columns: 'Compartment', 'Flow Name', 'Quantity', 'Unit', and 'DQI' (automated in Python)._ -### Output Flows +## Preferred Providers + +_Include a process or description of upstream providers for each of the product flows identified in the input table._ + +## Suggested Downstream Uses + +_Include a process or description of process, models, or domains where the use of this unit process may be applicable._ + +# Methods + +## Scenarios + +_For natural gas unit processes, there are often scenarios that may be described in more detail here._ + +## Allocation + +_State whether any allocation methods were used across co-products or by-production identified in the outputs table._ + + +## Calculations and Supplements -### Process Parameters - +_Include links and descriptions to additional resources that were utilized in creating this report and how they were utilized._ +_This includes, but is not limited to Excel workbooks, Python scripts, Jupyter notebooks, and other data files._ -### Allocation +## Modeling Assumptions + +_List any assumptions and known limitations to the utility or applicability of this unit process._ + +# Data + +## Data Quality + +_Provide an overview of the data quality index utilized for this process, and include a table of its scores (see below for example empty table)._ + +
+ +| DQI Indicator | Score | Basis for Score | +|--------------|:-----:|----------------| +| | | | +| | | | +| | | | +| | | | +| | | | + +
+ +## Data Source Information + +_Additional data source metadata is included here (e.g., Data Completeness, Data Selection, Data Treatment, Sampling Procedure, and Data Collection Period), which should all be automated from openLCA (see below for empty table example)._ + +
+ +| Element | Description | +|--------|-------------| +| Data Completeness | | +| Data Selection | | +| Data Treatment | | +| Sampling Procedure | | +| Data Collection Period | | + +
-### Calculations ## References +_List the references cited in this report following a standard style (e.g., APA or IEEE)._ + +# Use Information + +Use Advice +: _Describe the intended application and any additional uses._ + +Access and Restrictions +: Neither the U.S. Department of Energy (DOE) National Energy Technology Laboratory (NETL) nor any person acting on behalf of these organizations: + + A. Makes any warranty or representation, express or implied, with respect to the accuracy, completeness, or usefulness of the information contained in this document, or that the use of any information, apparatus, method, or process disclosed in this document may not infringe on privately owned rights; or + + B. Assumes any liability with this report as to its use, or damages resulting from the use of any information, apparatus, method, or process disclosed in this document. + + Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise, does not necessarily constitute or imply its endorsement, recommendation, or favoring by NETL. The views and opinions of the authors expressed herein do not necessarily state or reflect those of NETL. + # Document Control Information + Date Created -: TBA +: _The date when the report was finalized._ Point of Contact -: TBA +: _Name and email address._ Revision History -: TBA +: _Version number. If other than 1.0, consider including a list of changes._ How to Cite This Document -: TBA +: _Recommended citation (full text)._ + +Archived +: _True/False (Yes/No) to indicate the active (false) or superseded (true) status._ # Disclaimer/Terms of Use + +_Standard federal disclaimer (see default language below)._ + {self.disclaimer} """ @@ -558,8 +743,13 @@ def fetch_data(self, uuid=None): self.project_doc = self.format_project_doc(raw_process_doc) self.poc = self.format_poc(raw_process_doc) - param_list = self.netlolca.find_process_parameters(uuid) - self.param_table = self.format_parameter_table(param_list) + # NEW: EY25 test case uses the supplemental MD table [260219;TWD] + # param_list = self.netlolca.find_process_parameters(uuid) + # self.param_table = self.format_parameter_table(param_list) + self.param_table = '
' + self.param_table += '\n\n' + self.param_table += read_markdown_table("up_test_parameters.txt") + self.param_table += "\n\n
" sources = self.netlolca.get_sources(uuid, False) self.sources = [self.format_source(x) for x in sources] @@ -685,8 +875,11 @@ def format_flows(self, flows): comp = flows['category'][i] dq = flows['dq'][i] name = flows['name'][i] - # HOTFIX: rounds to 3 significant figures in scientific not. - quantity = "%0.3E" % flows['amount'][i] + # HOTFIX: for scenarios, use formula if available [260219; TWD] + quantity = flows['amountFormula'][i] + if quantity is None: + # HOTFIX: round to 3 sig figs in scientific notation + quantity = "%0.3E" % flows['amount'][i] unit = flows['unit'][i] flows_markdown += ( f"| {comp} | {name} | {quantity} | {unit} | {dq} |\n" @@ -762,6 +955,9 @@ def format_parameter_table(self, param_list): "The following table provides process and global parameter " "values and their associated uncertainty.\n\n" ) + #Open scrollable table + md_table += '
\n\n' + md_table += ( "| Scope | Name | Value | Uncertainty | Description |\n") md_table += ( @@ -812,6 +1008,10 @@ def format_parameter_table(self, param_list): md_table += "| " + p_string + " |\n" md_formula += p_form + "\n\n" + # Close scrollable table + if num_params > 0: + md_table += "\n
\n" + # Add the formulas before the parameter table. md_table = md_formula + md_table @@ -998,18 +1198,18 @@ def get_file_name(self, ext=".md"): if self.reference_name: # No spaces in file names p1 = re.compile("\\s+") - r_name = re.sub(p1, "_", self.reference_name) + r_name = re.sub(p1, "-", self.reference_name) # No special characters in file names p2 = re.compile("[/@.,&'\\\\\\(|\\)<>#;]+") - r_name = re.sub(p2, "_", r_name) + r_name = re.sub(p2, "-", r_name) - # Reduce extra underscores - p3 = re.compile("_+") - r_name = re.sub(p3, "_", r_name) + # Reduce extra dashes + p3 = re.compile("-+") + r_name = re.sub(p3, "-", r_name) - # Drop trailing underscore - if r_name[-1] == "_": + # Drop trailing dash + if r_name[-1] == "-": r_name = r_name[0:-1] r_name += ext @@ -1136,17 +1336,33 @@ def read_report_markdown(self, md_file=None): else: self.md = md_txt - def save_markdown(self): + def save_markdown(self, blank=False): """Generate markdown report and save to file. + Parameters + ---------- + blank : bool (optional) + Whether to save a blank template. + Defaults to false. + Notes ----- 1. The file is saved to the globally defined output directory with a file name that defaults to the reference process name. 2. This method re-creates the markdown report based on fetched data; any changes to the original markdown will be overwritten! + + Examples + -------- + >>> r = NetlOlcaReport(NetlOlca()) # initialize empty class + >>> r.save_markdown(blank=True) # create empty template """ - self.md = self.create_report_markdown() + if blank: + if self.reference_name is None: + self.reference_name = "blank-report" + self.md = self.empty_template() + else: + self.md = self.create_report_markdown() file_path = self.get_file_path('md') with open(file_path, 'w', encoding='utf-8', newline='\n') as f: f.write(self.md) @@ -1290,9 +1506,9 @@ def _fix_formula(p_name, f_txt, in_line=False): # Put between latex equation markers if in_line: - new_txt = "$%s%" % new_txt + new_txt = f"${new_txt}$" else: - new_txt = "$$\n%s\n$$" % new_txt + new_txt = f"$$\n{new_txt}\n$$" return new_txt @@ -1349,3 +1565,212 @@ def _fix_uncertainty(param): r_str = "%s (%s)" % (u_type, u_params) return r_str + + +def read_markdown_table(md_file): + """Helper function to read a markdown table from a file. + + Parameters + ---------- + md_file : str + Markdown file name. + Note: file must be located in the ``DATA_DIR`` directory. + + Returns + ------- + str + Markdown table as a Python string. + + Raises + ------ + OSError + If the file does not exist in the ``DATA_DIR``. + """ + file_path = os.path.join(DATA_DIR, md_file) + if not os.path.isfile(file_path): + raise OSError("Failed to find %s in %s!" % (md_file, DATA_DIR)) + with open(file_path, 'r') as f: + my_data = f.readlines() # newline character is in the string + my_str = "".join(my_data) + return my_str + + +def scenario_attribute_table( + workbook_filename: str, + sheet_name: str = "PS", + *, + header_row_excel: int = 6, + start_col_excel: str = "D", + max_height_px: int = 600, + return_dataframe: bool = False, +): + """Build scenario attribute table from scenario header strings in an Excel sheet. + + Workbook must be located in DATA_DIR. Scenario names are assumed to be in the + header row (Excel row header_row_excel) starting at start_col_excel. + """ + workbook_path = os.path.join(DATA_DIR, workbook_filename) + if not os.path.isfile(workbook_path): + raise OSError(f"Workbook not found in DATA_DIR: {workbook_path}") + + # Convert Excel col letter(s) to 0-indexed integer + col_letter = str(start_col_excel).strip().upper() + if not col_letter or any(not ("A" <= c <= "Z") for c in col_letter): + raise ValueError(f"Invalid Excel column letter: {start_col_excel}") + + start_col_index = 0 + for c in col_letter: + start_col_index = start_col_index * 26 + (ord(c) - ord("A") + 1) + start_col_index -= 1 + + header_row_index = header_row_excel - 1 + scenario_source_df = pd.read_excel( + workbook_path, + sheet_name=sheet_name, + header=header_row_index, + engine="openpyxl", + ) + + scenario_header_candidates = list(scenario_source_df.columns[start_col_index:]) + scenario_names = [] + for header_val in scenario_header_candidates: + if pd.isna(header_val): + continue + header_text = re.sub(r"\s+", " ", str(header_val)).strip().strip(",") + if header_text.startswith("External Combustion Boilers") or header_text.startswith("Internal Combustion Engines"): + scenario_names.append(header_text) + + scenario_records = [] + for scenario_id, scenario_name in enumerate(scenario_names, start=1): + name = re.sub(r"\s+", " ", str(scenario_name)).strip().strip(",") + parts = [p.strip() for p in name.split(",") if p.strip()] + + tech = parts[0] if len(parts) > 0 else "NA" + sector = parts[1] if len(parts) > 1 else "NA" + fuel = parts[2] if len(parts) > 2 else "NA" + remainder = parts[3:] if len(parts) > 3 else [] + + regulatory_vintage = "NA" + size_class_mmbtu_hr = "NA" + firing_type = "NA" + operating_mode = "Standard" + + if "External Combustion Boilers" in tech: + technology_family = "External Combustion Boiler" + equipment_category = "Boiler" + + if "Pre-NSPS" in name: + regulatory_vintage = "Pre-NSPS" + elif "Post" in name and "NSPS" in name: + regulatory_vintage = "Post-NSPS" + + if any(r.lower() == "cogeneration" for r in remainder): + operating_mode = "Cogeneration" + equipment_subtype = "Cogeneration" + control_technology = remainder[-1] if remainder else "NA" + + scenario_records.append({ + "Scenario_ID": scenario_id, + "Scenario_Name": name, + "Technology_Family": technology_family, + "Sector": sector, + "Fuel": fuel, + "Equipment_Category": equipment_category, + "Equipment_Subtype": equipment_subtype, + "Size_Class_MMBtu_hr": "NA", + "Firing_Type": "NA", + "Control_Technology": control_technology, + "Regulatory_Vintage": regulatory_vintage, + "Operating_Mode": operating_mode, + }) + continue + + if any("Tangentially Fired Units" in r for r in remainder): + equipment_subtype = "Tangential" + firing_type = "Tangential" + else: + equipment_subtype = "Non-tangential" + firing_type = "Non-tangential" + joined = " ".join(remainder) + if re.search(r">\s*100", joined): + size_class_mmbtu_hr = ">100" + elif re.search(r"<\s*100", joined): + size_class_mmbtu_hr = "<100" + elif re.search(r"10\s*-\s*100", joined): + size_class_mmbtu_hr = "10-100" + elif re.search(r"<\s*10", joined): + size_class_mmbtu_hr = "<10" + + control_technology = remainder[-1] if remainder else "NA" + + else: + technology_family = "Internal Combustion Engine" + equipment_category = "Engine" + equipment_subtype = remainder[0] if remainder else "NA" + control_technology = remainder[-1] if remainder else "NA" + operating_mode = "Cogeneration" if ":Cogeneration" in equipment_subtype else "Standard" + regulatory_vintage = "NA" + size_class_mmbtu_hr = "NA" + firing_type = "NA" + + scenario_records.append({ + "Scenario_ID": scenario_id, + "Scenario_Name": name, + "Technology_Family": technology_family, + "Sector": sector, + "Fuel": fuel, + "Equipment_Category": equipment_category, + "Equipment_Subtype": equipment_subtype, + "Size_Class_MMBtu_hr": size_class_mmbtu_hr, + "Firing_Type": firing_type, + "Control_Technology": control_technology, + "Regulatory_Vintage": regulatory_vintage, + "Operating_Mode": operating_mode, + }) + + scenario_columns = [ + "Scenario_ID", + "Scenario_Name", + "Technology_Family", + "Sector", + "Fuel", + "Equipment_Category", + "Equipment_Subtype", + "Size_Class_MMBtu_hr", + "Firing_Type", + "Control_Technology", + "Regulatory_Vintage", + "Operating_Mode", + ] + scenario_table_df = pd.DataFrame(scenario_records) + for c in scenario_columns: + if c not in scenario_table_df.columns: + scenario_table_df[c] = "NA" + scenario_table_df = scenario_table_df[scenario_columns].fillna("NA") + + scenario_table_md = ( + f'
\n\n' + f'{scenario_table_df.to_markdown(index=False)}\n\n' + f'
\n' + ) + + if return_dataframe: + return scenario_table_df, scenario_table_md + return scenario_table_md + + +# +# SANDBOX +# +if __name__ == '__main__': + from up_template import NetlOlcaReport + from netlolca import NetlOlca + n = NetlOlca() # initialize data manager + n.connect() # establish with IPC Server + n.read() # read openLCA database + r = NetlOlcaReport(n) # creates 'output' folder + ps_id = r.netlolca.get_spec_ids( # find 1st product system UUID + r.netlolca.get_spec_class("Product system") + )[0] + r.fetch_data(ps_id) # automated data scraping + r.convert_to_html() # create .md; render to .html