diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 478eea8..37ab521 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,16 +2,14 @@ name: docs on: push: - branches: [master, main] - #pull_request: TODO: Enable me - # branches:[master, main] + branches: [main] jobs: tests: name: "Build docs" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python id: setup_python diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6bd9bdd..c3ed29b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,13 +2,13 @@ name: tests on: push: - branches: [master, main] + branches: [main] pull_request: - branches: [master, main] + branches: [main] jobs: tests: - name: "Run tests on py${{ matrix.python-version }}" + name: 'Run tests on py${{ matrix.python-version }}' runs-on: ubuntu-latest strategy: fail-fast: false @@ -18,9 +18,6 @@ jobs: - python-version: "3.7" - python-version: "3.8" - python-version: "3.9" - - python-version: "3.10" - - python-version: "3.11" - - python-version: "3.12" steps: - uses: actions/checkout@v2 @@ -35,7 +32,7 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} - - name: Install vplanet-private pre-release + - name: Install VPLanet id: install if: steps.setup_python.outcome == 'success' shell: bash -l {0} @@ -44,7 +41,6 @@ jobs: run: | python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet.git@main#egg=vplanet - - name: Install bigplanet id: tools if: steps.install.outcome == 'success' @@ -73,7 +69,7 @@ jobs: comment_mode: update last - name: CodeCov - uses: codecov/codecov-action@v1.5.0 + uses: codecov/codecov-action@v2.1.0 with: files: ./coverage.xml #shell: bash -l {0} diff --git a/.gitignore b/.gitignore index 1e476b1..ee4243d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,29 +21,6 @@ SeasonalClimateFiles *.csv *.md5 -#files created from running tests -tests/Bpstatus/BP_Status/ -tests/Bpstatus/.* -tests/CreateHDF5/BP_CreateHDF5/ -tests/CreateHDF5/.* -tests/ExtractArchive/BP_Extract/ -tests/ExtractArchive/.* -tests/Stats/BP_Stats/ -tests/Stats/.* -tests/ExtractFilterArchive/BP_Extract/ -tests/ExtractFilterArchive/.* -tests/ExtractFilterRawData/BP_Extract/ -tests/ExtractFilterRawData/.* -tests/MD5CheckSum/BP_Extract/ -tests/MD5CheckSum/.* -tests/UlyssesAggregated/BP_Extract/ -tests/UlyssesAggregated/.* -tests/UlyssesForward/BP_Extract/ -tests/UlyssesForward/.* -tests/SingleSim/BP_Extract/ -tests/SingleSim/.* - - .coverage # Aux files / build info / garbage @@ -89,3 +66,4 @@ junit/test-results.xml coverage.xml tests/UlyssesAggregated/User.csv +htmlcov/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d7ebec1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +exclude: 'docs/' + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/psf/black + rev: 21.12b0 + hooks: + - id: black + verbose: true + exclude: '__init__.py' + + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + args: ['--profile', 'black'] + verbose: true + exclude: '__init__.py' diff --git a/README.md b/README.md index 544e953..b0333f0 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,15 @@

- - + + + + - - - - + + +

-A suite of data analysis and extraction routines for [VPLanet](https://github.com/VirtualPlanetaryLaboratory/vplanet) and designed to be used with [VSPACE](https://github.com/VirtualPlanetaryLaboratory/vspace) and [MultiPlanet](https://github.com/VirtualPlanetaryLaboratory/multi-planet) scripts. It can also generate files for use with the free version of the [VR Ulysses data explorer](https://www.vrulysses.com/download-ulysses). [Read the docs](https://VirtualPlanetaryLaboratory.github.io/bigplanet/) to learn how to get the most out of your VPLanet parameter sweeps. +A suite of data analysis and extraction routines for [VPLanet](https://github.com/VirtualPlanetaryLaboratory/vplanet) and designed to be used with [VSPACE](https://github.com/VirtualPlanetaryLaboratory/vspace) and [MultiPlanet](https://github.com/VirtualPlanetaryLaboratory/multi-planet) scripts. It can also generate files for use with the free version of the [VR Ulysses data explorer](https://www.vrulysses.com/download-ulysses). [Read the docs](https://VirtualPlanetaryLaboratory.github.io/bigplanet/) to learn how to get the most out of your `VPLanet` parameter sweeps. diff --git a/bigplanet/bigplanet.py b/bigplanet/bigplanet.py index 97e566c..254283e 100644 --- a/bigplanet/bigplanet.py +++ b/bigplanet/bigplanet.py @@ -1,19 +1,30 @@ #!/usr/bin/env python -import os +import argparse import multiprocessing as mp +import os import subprocess as sub -import argparse + import h5py import numpy as np import pandas as pd + from .bigplanet_archive import Archive from .bigplanet_filter import Filter -from .bp_get import ReadFile from .bp_extract import Md5CheckSum +from .bp_get import ReadFile -def Main(bpInputFile, cores, quiet, overwrite, verbose, archive, deleterawdata, ignorecorrupt): +def Main( + bpInputFile, + cores, + quiet, + overwrite, + verbose, + archive, + deleterawdata, + ignorecorrupt, +): # folder,bplArchive,output,bodyFileList,primaryFile,IncludeList,ExcludeList,Ulysses = ReadFile(file,verbose) # # if if IncludeList != [] and ExcludeList != [] and os.path.isfile(bplArchive) == False: @@ -23,14 +34,26 @@ def Main(bpInputFile, cores, quiet, overwrite, verbose, archive, deleterawdata, # print("Creating filtered BPF file...") # MainMethodF(bpInputFile,quiet,verbose) if deleterawdata == True: - #folder_name, bpl_file, outputFile, bodylist, primaryFile, includelist, excludelist, Ulysses, SimName - folder, bplArchive, output, bodyFileList, primaryFile, IncludeList, ExcludeList, Ulysses, simname = ReadFile( - bpInputFile, verbose, archive) + # folder_name, bpl_file, outputFile, bodylist, primaryFile, includelist, excludelist, Ulysses, SimName + ( + folder, + bplArchive, + output, + bodyFileList, + primaryFile, + IncludeList, + ExcludeList, + Ulysses, + simname, + ) = ReadFile(bpInputFile, verbose, archive) if os.path.exists(bplArchive) == True: Md5CheckSum(bplArchive, ignorecorrupt) reply = None - question = ("WARNING: This command will delete the " + - folder + " directory along with any MultiPlanet checkpoint files.") + question = ( + "WARNING: This command will delete the " + + folder + + " directory along with any MultiPlanet checkpoint files." + ) while reply not in ("y", "n"): reply = str(input(question + " (y/n): ")).lower().strip() if reply[:1] == "y": @@ -42,7 +65,7 @@ def Main(bpInputFile, cores, quiet, overwrite, verbose, archive, deleterawdata, if reply[:1] == "n": print("Files NOT deleted. Exiting.") exit() - if reply[:1] != "n" or reply[:1] != "y": + else: print("User input was not valid. Exiting.") exit() @@ -60,27 +83,63 @@ def Main(bpInputFile, cores, quiet, overwrite, verbose, archive, deleterawdata, def Arguments(): max_cores = mp.cpu_count() parser = argparse.ArgumentParser( - description="Extract data from Vplanet simulations") + description="Extract data from Vplanet simulations" + ) + parser.add_argument( + "bpInputFile", help="Name of the biugplanet input file" + ) + parser.add_argument( + "-c", + "--cores", + type=int, + default=max_cores, + help="Number of processors used", + ) + parser.add_argument( + "-o", + "--overwrite", + action="store_true", + help="overwrite file if it already exists", + ) + parser.add_argument( + "-a", + "--archive", + action="store_true", + help="flag for archive file creation", + ) + parser.add_argument( + "-deleterawdata", + "--deleterawdata", + action="store_true", + help="removes source files after creation of Bigplanet files", + ) parser.add_argument( - "bpInputFile", help="Name of the biugplanet input file") - parser.add_argument("-c", "--cores", type=int, - default=max_cores, help="Number of processors used") - parser.add_argument("-o", "--overwrite", action="store_true", - help="overwrite file if it already exists") - parser.add_argument("-a", "--archive", action="store_true", - help="flag for archive file creation") - parser.add_argument("-deleterawdata", "--deleterawdata", action="store_true", - help="removes source files after creation of Bigplanet files") - parser.add_argument("-ignorecorrupt", "--ignorecorrupt", action="store_true", - help="ignore data corruption for MD5 Checksum") + "-ignorecorrupt", + "--ignorecorrupt", + action="store_true", + help="ignore data corruption for MD5 Checksum", + ) # adds the quiet and verbose as mutually exclusive groups group = parser.add_mutually_exclusive_group() - group.add_argument("-q", "--quiet", action="store_true", - help="no output for bigplanet") - group.add_argument("-v", "--verbose", action="store_true", - help="Prints out excess output for bigplanet") + group.add_argument( + "-q", "--quiet", action="store_true", help="no output for bigplanet" + ) + group.add_argument( + "-v", + "--verbose", + action="store_true", + help="Prints out excess output for bigplanet", + ) args = parser.parse_args() - Main(args.bpInputFile, args.cores, args.quiet, - args.overwrite, args.verbose, args.archive, args.deleterawdata, args.ignorecorrupt) + Main( + args.bpInputFile, + args.cores, + args.quiet, + args.overwrite, + args.verbose, + args.archive, + args.deleterawdata, + args.ignorecorrupt, + ) diff --git a/bigplanet/bigplanet_archive.py b/bigplanet/bigplanet_archive.py index 858c7ba..f9309f6 100755 --- a/bigplanet/bigplanet_archive.py +++ b/bigplanet/bigplanet_archive.py @@ -1,22 +1,33 @@ #!/usr/bin/env python -import os +import argparse import multiprocessing as mp +import os import subprocess as sub -import argparse + import h5py import numpy as np import pandas as pd -from .bp_process import * -from .bp_get import * + from .bp_extract import * +from .bp_get import * +from .bp_process import * def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): # Get the directory and list of from the bpl file - dest_folder, bpl_file, outputFile, bodylist, primaryFile, includelist, excludelist, ulysses, SimName = ReadFile( - bpInputFile, verbose, archive=True) + ( + dest_folder, + bpl_file, + outputFile, + bodylist, + primaryFile, + includelist, + excludelist, + ulysses, + SimName, + ) = ReadFile(bpInputFile, verbose, archive=True) # we need to combine the body and primary files into one list infile_list = [] for i in bodylist: @@ -34,7 +45,7 @@ def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): vplanet_help = GetVplanetHelp() # creates the chepoint file name - checkpoint_file = os.getcwd() + '/' + '.' + dest_folder + '_BPL' + checkpoint_file = os.getcwd() + "/" + "." + dest_folder + "_BPL" # Create the checkpoint file to be used to keep track of the groups if os.path.isfile(checkpoint_file) == False: @@ -43,8 +54,9 @@ def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): # if it does exist, it checks for any 0's (sims that didn't complete) and # changes them to -1 to be re-ran else: - ReCreateCP(checkpoint_file, bpInputFile, - quiet, sim_list, dest_folder, force) + ReCreateCP( + checkpoint_file, bpInputFile, quiet, sim_list, dest_folder, force + ) # now that we have everything we need # we save the name of the Master HDF5 file @@ -56,8 +68,23 @@ def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): # for each core, create a process that adds a group to the hdf5 file and adds that to the Master HDF5 file for i in range(cores): - workers.append(mp.Process(target=par_worker, - args=(checkpoint_file, system_name, body_list, log_file, infile_list, quiet, lock, vplanet_help, master_hdf5_file, verbose))) + workers.append( + mp.Process( + target=par_worker, + args=( + checkpoint_file, + system_name, + body_list, + log_file, + infile_list, + quiet, + lock, + vplanet_help, + master_hdf5_file, + verbose, + ), + ) + ) for w in workers: w.start() for w in workers: @@ -67,46 +94,49 @@ def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): def CreateCP(checkpoint_file, input_file, sims): - with open(checkpoint_file, 'w') as cp: - cp.write('Vspace File: ' + os.getcwd() + '/' + input_file + '\n') - cp.write('Total Number of Simulations: ' + str(len(sims)) + '\n') + with open(checkpoint_file, "w") as cp: + cp.write("Vspace File: " + os.getcwd() + "/" + input_file + "\n") + cp.write("Total Number of Simulations: " + str(len(sims)) + "\n") for f in range(len(sims)): cp.write(sims[f] + " " + "-1 \n") - cp.write('THE END \n') + cp.write("THE END \n") def ReCreateCP(checkpoint_file, input_file, quiet, sims, folder_name, force): datalist = [] - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r") as f: for newline in f: if newline: datalist.append(newline.strip().split()) for l in datalist: - if l[1] == '0': - l[1] = '-1' + if l[1] == "0": + l[1] = "-1" folder = l[0] - with h5py.File(folder_name + '.bpa', "a") as master: - group_name = "/" + folder.split('/')[-1] + with h5py.File(folder_name + ".bpa", "a") as master: + group_name = "/" + folder.split("/")[-1] if group_name in master: if quiet == False: - print("Deleting", group_name, - "from BPL file...") + print( + "Deleting", + group_name, + "from BPL file...", + ) del master[group_name] - with open(checkpoint_file, 'w') as f: + with open(checkpoint_file, "w") as f: for newline in datalist: - f.writelines(' '.join(newline)+'\n') + f.writelines(" ".join(newline) + "\n") - if all(l[1] == '1' for l in datalist[2:-2]) == True: + if all(l[1] == "1" for l in datalist[2:-2]) == True: if quiet == False: print("All Groups in BPL file exist") if force == True: if quiet == False: print("Deleting BPL file...") - os.remove(folder_name + '.bpa') + os.remove(folder_name + ".bpa") if quiet == False: print("Deleting checkpoint file...") os.remove(checkpoint_file) @@ -116,10 +146,21 @@ def ReCreateCP(checkpoint_file, input_file, quiet, sims, folder_name, force): else: if quiet == False: - print('Continuing from Checkpoint...') - - -def par_worker(checkpoint_file, system_name, body_list, log_file, in_files, quiet, lock, vplanet_help, h5_file, verbose): + print("Continuing from Checkpoint...") + + +def par_worker( + checkpoint_file, + system_name, + body_list, + log_file, + in_files, + quiet, + lock, + vplanet_help, + h5_file, + verbose, +): while True: @@ -127,25 +168,25 @@ def par_worker(checkpoint_file, system_name, body_list, log_file, in_files, quie datalist = [] data = {} - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r") as f: for newline in f: datalist.append(newline.strip().split()) - folder = '' + folder = "" for l in datalist: - if l[1] == '-1': + if l[1] == "-1": folder = l[0] - l[1] = '0' + l[1] = "0" break if not folder: lock.release() return - with open(checkpoint_file, 'w') as f: + with open(checkpoint_file, "w") as f: for newline in datalist: - f.writelines(' '.join(newline)+'\n') + f.writelines(" ".join(newline) + "\n") lock.release() @@ -154,35 +195,49 @@ def par_worker(checkpoint_file, system_name, body_list, log_file, in_files, quie lock.acquire() datalist = [] - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r") as f: for newline in f: datalist.append(newline.strip().split()) - group_name = "/" + folder.split('/')[-1] + group_name = "/" + folder.split("/")[-1] # creates the bpl file and reads to make sure the group name is in the file - with h5py.File(h5_file, 'a') as Master: + with h5py.File(h5_file, "a") as Master: # if not then add it if group_name not in Master: if quiet == False: print("Creating", group_name, "...") - data = GatherData(data, system_name, body_list, - log_file, in_files, vplanet_help, folder, verbose) - DictToBP(data, vplanet_help, Master, - verbose, group_name, archive=True) + data = GatherData( + data, + system_name, + body_list, + log_file, + in_files, + vplanet_help, + folder, + verbose, + ) + DictToBP( + data, + vplanet_help, + Master, + verbose, + group_name, + archive=True, + ) for l in datalist: if l[0] == folder: - l[1] = '1' + l[1] = "1" break else: for l in datalist: if l[0] == folder: - l[1] = '1' + l[1] = "1" break - with open(checkpoint_file, 'w') as f: + with open(checkpoint_file, "w") as f: for newline in datalist: - f.writelines(' '.join(newline)+'\n') + f.writelines(" ".join(newline) + "\n") lock.release() diff --git a/bigplanet/bigplanet_filter.py b/bigplanet/bigplanet_filter.py index b1f9977..fa7725b 100755 --- a/bigplanet/bigplanet_filter.py +++ b/bigplanet/bigplanet_filter.py @@ -1,18 +1,20 @@ #!/usr/bin/env python -import os -import multiprocessing as mp -import subprocess as sub import argparse -import h5py -import numpy as np import csv +import multiprocessing as mp +import os import pathlib +import subprocess as sub import sys + +import h5py +import numpy as np import pandas as pd -from .bp_process import * -from .bp_get import * + from .bp_extract import * +from .bp_get import * +from .bp_process import * def SplitsaKey(saKeylist, verbose): @@ -26,13 +28,23 @@ def SplitsaKey(saKeylist, verbose): # loop over the key list for item in saKeylist: # to figure out what list they belong in, we have to rpartion them and look at the last word - spl = item.rpartition(':') - if spl[-1] == "initial" or spl[-1] == "final" or spl[-1] == 'OutputOption' or spl[-1] == 'GridOutputOption': + spl = item.rpartition(":") + if ( + spl[-1] == "initial" + or spl[-1] == "final" + or spl[-1] == "OutputOption" + or spl[-1] == "GridOutputOption" + ): loglist.append(item) # check if its forward or any of the statsitical functions elif ( - spl[-1] == "forward" or spl[-1] == "mean" or spl[-1] == "mode" or spl[-1] == "max" or - spl[-1] == "min" or spl[-1] == "geomean" or spl[-1] == "stddev" + spl[-1] == "forward" + or spl[-1] == "mean" + or spl[-1] == "mode" + or spl[-1] == "max" + or spl[-1] == "min" + or spl[-1] == "geomean" + or spl[-1] == "stddev" ): forwardlist.append(item) # checks if its a body file @@ -54,8 +66,17 @@ def SplitsaKey(saKeylist, verbose): def Filter(file, quiet, verbose, ignorecorrupt, override): - folder, bplArchive, output, bodyFileList, primaryFile, IncludeList, ExcludeList, Ulysses, SimName = ReadFile( - file, verbose=True, archive=False) + ( + folder, + bplArchive, + output, + bodyFileList, + primaryFile, + IncludeList, + ExcludeList, + Ulysses, + SimName, + ) = ReadFile(file, verbose=True, archive=False) vplHelp = GetVplanetHelp() @@ -66,20 +87,32 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): infile_list.append(primaryFile) if os.path.exists(output) and Ulysses == 0: - print("ERROR:", output, "already exists. Please delete", - output, "and try again") + print( + "ERROR:", + output, + "already exists. Please delete", + output, + "and try again", + ) elif os.path.exists(output) and override == True: print("Overriding output file...") sub.run(["rm", output]) # if the bpl archive file does NOT exist, we have to get the data manually if os.path.isfile(bplArchive) == False: - print("WARNING: BPA File does not exist. Obtaining data from source folder. This make take some time...") + print( + "WARNING: BPA File does not exist. Obtaining data from source folder. This make take some time..." + ) # first we need to see what keys go to what vplanet file (ie body file or log file or forward file) if IncludeList: - loglist, optionList, forwardlist, climatelist, backwardlist = SplitsaKey( - IncludeList, verbose) + ( + loglist, + optionList, + forwardlist, + climatelist, + backwardlist, + ) = SplitsaKey(IncludeList, verbose) # now that we have the list of which keys to look for in which files, we can process the files and grab the data if SimName: simList = GetSims(folder, simname=SimName) @@ -97,7 +130,8 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): print("Processing Log file", log_file) # we need to get the system name to get the name of the logfile data = ProcessLogFile( - log_file, data, sim, verbose, incl=IncludeList) + log_file, data, sim, verbose, incl=IncludeList + ) print(data) if optionList: @@ -106,7 +140,8 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): if verbose: print("Processing input file", k) data = ProcessInputfile( - data, k, sim, vplHelp, verbose, incl=IncludeList) + data, k, sim, vplHelp, verbose, incl=IncludeList + ) if forwardlist: print("Forward file data requested") for body in body_names: @@ -121,17 +156,28 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): if outfile in data: forward_name = data[outfile] else: - forward_name = system_name + '.' + body + '.forward' + forward_name = ( + system_name + "." + body + ".forward" + ) - header = [body + ':' + 'OutputOrder'] + header = [body + ":" + "OutputOrder"] heading = {} print("Obtaining Header for Logfile...") heading = ProcessLogFile( - log_file, heading, sim, verbose, incl=header) + log_file, heading, sim, verbose, incl=header + ) print("Processing Forward File", forward_name) data = ProcessOutputfile( - forward_name, data, body, heading, ':forward', sim, verbose, incl=IncludeList) + forward_name, + data, + body, + heading, + ":forward", + sim, + verbose, + incl=IncludeList, + ) if backwardlist: print("Processing Backwards File") @@ -145,43 +191,72 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): if outfile in data: backward_name = data[outfile] else: - backward_name = system_name + '.' + body + '.backward' + backward_name = ( + system_name + "." + body + ".backward" + ) - header = [body + ':' + 'OutputOrder'] + header = [body + ":" + "OutputOrder"] heading = {} heading = ProcessLogFile( - log_file, heading, sim, verbose, incl=header) + log_file, heading, sim, verbose, incl=header + ) data = ProcessOutputfile( - backward_name, data, body, heading, ':backward', sim, verbose, incl=IncludeList) + backward_name, + data, + body, + heading, + ":backward", + sim, + verbose, + incl=IncludeList, + ) if climatelist: for body in body_names: if any(body in s for s in climatelist) == False: continue else: - header = [body + ':' + 'GridOutputOrder'] - climate_name = system_name + '.' + body + '.Climate' + header = [body + ":" + "GridOutputOrder"] + climate_name = ( + system_name + "." + body + ".Climate" + ) heading = {} heading = ProcessLogFile( - log_file, heading, sim, verbose, incl=header) + log_file, heading, sim, verbose, incl=header + ) data = ProcessOutputfile( - climate_name, data, body, heading, ':climate', sim, verbose, incl=IncludeList) + climate_name, + data, + body, + heading, + ":climate", + sim, + verbose, + incl=IncludeList, + ) if Ulysses == 1: DictToCSV(data, ulysses=True) else: - with h5py.File(output, 'w') as filter: + with h5py.File(output, "w") as filter: # Change this to DictToBP <- this reads from Dict to Bigplanet File - DictToBP(data, vplHelp, filter, verbose, - group_name=None, archive=False) + DictToBP( + data, + vplHelp, + filter, + verbose, + group_name=None, + archive=False, + ) # if the bpl file DOES exist, we just need to open it and extract the data to put it in the filter file else: print("Extracting data from BPA File. Please wait...") archive = BPLFile(bplArchive, ignorecorrupt) if Ulysses == 1: if SimName: - ArchiveToCSV(archive, IncludeList, output, - ulysses=1, group=SimName) + ArchiveToCSV( + archive, IncludeList, output, ulysses=1, group=SimName + ) else: # Change this to ArchiveToCSV <- this reads from archive file and exports a CSV ArchiveToCSV(archive, IncludeList, output, ulysses=1) diff --git a/bigplanet/bp_extract.py b/bigplanet/bp_extract.py index fa970f5..4adf819 100755 --- a/bigplanet/bp_extract.py +++ b/bigplanet/bp_extract.py @@ -1,28 +1,30 @@ #!/usr/bin/env python -import multiprocessing as mp -import h5py -import numpy as np -from scipy import stats -import statistics as st import csv -import pandas as pd import hashlib +import multiprocessing as mp import os +import statistics as st import sys + +import h5py +import numpy as np +import pandas as pd +from scipy import stats + from .bp_get import GetVplanetHelp from .bp_process import DictToBP def BPLFile(hf, ignore_corrupt=False): - file = h5py.File(hf, 'r') + file = h5py.File(hf, "r") key_list = list(file.keys()) if ":" not in key_list[0]: Md5CheckSum(hf, ignore_corrupt) - return h5py.File(hf, 'r') + return h5py.File(hf, "r") def ExtractColumn(hf, k): @@ -38,8 +40,8 @@ def ExtractColumn(hf, k): k : str the name of the column that is to be extracted Example: - k = 'earth_Obliquity_final' - The syntax of the column names is body_variable_aggregation + k = 'earth:Obliquity:final' + The syntax of the column names is body:variable:aggregation the lists of aggregations (and how to call them) is as follows: forward file data (forward), initial data (initial), @@ -66,24 +68,24 @@ def ExtractColumn(hf, k): var = k.split(":")[1] - if var == 'OutputOrder' or var == 'GridOutputOrder': + if var == "OutputOrder" or var == "GridOutputOrder": if archive == True: - dataset = hf[key_list[0] + '/' + k] + dataset = hf[key_list[0] + "/" + k] for d in dataset: for value in d: - data.append(value.decode('UTF-8')) + data.append(value.decode("UTF-8")) else: for v in hf[k]: for item in v: - data.append(item.decode('UTF-8')) + data.append(item.decode("UTF-8")) else: aggreg = k.split(":")[2] - if aggreg == 'forward' or aggreg == "backward" or aggreg == "climate": + if aggreg == "forward" or aggreg == "backward" or aggreg == "climate": if archive == True: for key in key_list: - dataset = hf[key + '/' + k] + dataset = hf[key + "/" + k] for d in dataset: data.append(d) else: @@ -91,46 +93,46 @@ def ExtractColumn(hf, k): for d in dataset: data.append(d) - elif aggreg == 'mean': + elif aggreg == "mean": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((st.mean(i))) - elif aggreg == 'stddev': + elif aggreg == "stddev": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((st.stdev(i))) - elif aggreg == 'min': + elif aggreg == "min": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((min(i))) - elif aggreg == 'max': + elif aggreg == "max": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((max(i))) - elif aggreg == 'mode': + elif aggreg == "mode": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((stats.mode(i))) - elif aggreg == 'geomean': + elif aggreg == "geomean": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((stats.gmean(i))) - elif aggreg == 'initial' or aggreg == 'final' or aggreg == 'option': + elif aggreg == "initial" or aggreg == "final" or aggreg == "option": if archive == True: for key in key_list: - dataset = hf[key + '/' + k] + dataset = hf[key + "/" + k] for d in dataset: data.append(float(d)) else: @@ -138,7 +140,7 @@ def ExtractColumn(hf, k): data.append(float(d)) else: - print('ERROR: Uknown aggregation option: ', aggreg) + print("ERROR: Uknown aggregation option: ", aggreg) exit() return data @@ -176,20 +178,20 @@ def ExtractUnits(hf, k): key_list = list(hf.keys()) if ":" not in key_list[0]: - dataset = hf[key_list[0] + '/' + k] + dataset = hf[key_list[0] + "/" + k] else: dataset = hf[k] - return dataset.attrs.get('Units') + return dataset.attrs.get("Units") def ForwardData(hf, k): data = [] key_list = list(hf.keys()) - forward = k.rpartition(':')[0] + ':forward' + forward = k.rpartition(":")[0] + ":forward" # if hf is an archive file if ":" not in key_list[0]: for key in key_list: - dataset = hf[key + '/' + forward] + dataset = hf[key + "/" + forward] for d in dataset: data.append(d) # for v in d: @@ -210,13 +212,13 @@ def HFD5Decoder(dataset): if "forward" in dataset.name: for value in d: for num in value: - num = num.astype(float, casting='safe') - #data.append(value.astype(float, casting='safe')) + num = num.astype(float, casting="safe") + # data.append(value.astype(float, casting='safe')) else: - d = d.astype(float, casting='safe') - # and now we reshape it the same shape as the original dataset - #shape = dataset.shape - #data = np.reshape(data, shape) + d = d.astype(float, casting="safe") + # and now we reshape it the same shape as the original dataset + # shape = dataset.shape + # data = np.reshape(data, shape) # print(data) # data.tolist() @@ -251,7 +253,7 @@ def ExtractUniqueValues(hf, k): if archive == True: for key in key_list: - dataset = hf[key + '/' + k] + dataset = hf[key + "/" + k] for d in dataset: data.append(float(d.decode("UTF-8"))) else: @@ -290,7 +292,9 @@ def CreateMatrix(xaxis, yaxis, zarray, orientation=1): ynum = len(yaxis) if xnum * ynum != len(zarray): - print("ERROR: Cannot reshape zarray into shape (", xnum, ",", ynum, ")") + print( + "ERROR: Cannot reshape zarray into shape (", xnum, ",", ynum, ")" + ) exit() zmatrix = np.reshape(zarray, (ynum, xnum)) @@ -354,12 +358,19 @@ def ArchiveToFiltered(inputfile, columns, exportfile): units[i] = ExtractUnits(inputfile, i) with h5py.File(exportfile, "a") as f_dest: - f_dest.create_dataset( - i, data=export[i]) - f_dest[i].attrs['Units'] = units[i] - - -def ArchiveToCSV(inputfile, columns, exportfile, delim=" ", header=False, ulysses=0, group=None): + f_dest.create_dataset(i, data=export[i]) + f_dest[i].attrs["Units"] = units[i] + + +def ArchiveToCSV( + inputfile, + columns, + exportfile, + delim=" ", + header=False, + ulysses=0, + group=None, +): """ Writes an Output file in csv format @@ -402,11 +413,11 @@ def ArchiveToCSV(inputfile, columns, exportfile, delim=" ", header=False, ulysse export.append(i) if ulysses == 1: - delim = ',' - exportfile = 'User.csv' + delim = "," + exportfile = "User.csv" if delim == "": - print('ERROR: Delimiter cannot be empty') + print("ERROR: Delimiter cannot be empty") exit() with open(exportfile, "w", newline="") as f: @@ -421,13 +432,15 @@ def ArchiveToCSV(inputfile, columns, exportfile, delim=" ", header=False, ulysse if header == True: writer.writerow(columns) - export = np.array(export, dtype='object').T.tolist() + export = np.array(export, dtype="object").T.tolist() for name in export: for data in name: writer.writerow([data]) -def DictToCSV(dictData, exportfile="bigplanet.csv", delim=" ", header=False, ulysses=0): +def DictToCSV( + dictData, exportfile="bigplanet.csv", delim=" ", header=False, ulysses=0 +): """ Writes an Output file in csv format @@ -459,9 +472,9 @@ def DictToCSV(dictData, exportfile="bigplanet.csv", delim=" ", header=False, uly """ if ulysses == 1: - delim = ',' + delim = "," # headers.append("") - exportfile = 'User.csv' + exportfile = "User.csv" headers = [] for k in dictData.keys(): @@ -475,10 +488,10 @@ def DictToCSV(dictData, exportfile="bigplanet.csv", delim=" ", header=False, uly dictData[k] = [item for sublist in v for item in sublist] if delim == "": - print('ERROR: Delimiter cannot be empty') + print("ERROR: Delimiter cannot be empty") exit() - df = pd.DataFrame.from_dict(dictData, orient='index').transpose() + df = pd.DataFrame.from_dict(dictData, orient="index").transpose() if header == True or ulysses == 1: df.to_csv(exportfile, index=False, header=True) @@ -519,6 +532,7 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): md5file = name + ".md5" # if it doesn't exist, we need to create it if os.path.isfile(md5file) == False: + print(md5file+" file not found. Generating.") with open(md5file, "w") as md5: with open(bpa, "rb") as f: file_hash = hashlib.md5() @@ -526,13 +540,14 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): # while chunk := f.read(32768): # file_hash.update(chunk) # if sys.version_info <= (3, 7): - for chunk in iter(lambda: f.read(32768), b''): + for chunk in iter(lambda: f.read(32768), b""): file_hash.update(chunk) - #print("MD5 Checksum:", file_hash.hexdigest()) + # print("MD5 Checksum:", file_hash.hexdigest()) md5.write(file_hash.hexdigest()) else: with open(md5file, "r") as md5: + #print("Found "+md5file) md5_old = md5.readline() #print("MD5 from " + md5file + ":", md5_old) with open(bpa, "rb") as f: @@ -541,17 +556,19 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): # while chunk := f.read(32768): # file_hash.update(chunk) # if sys.version_info <= (3, 7): - for chunk in iter(lambda: f.read(32768), b''): + for chunk in iter(lambda: f.read(32768), b""): file_hash.update(chunk) new_md5 = file_hash.hexdigest() #print("MD5 from " + name + ".bpa: " + new_md5) if md5_old == new_md5: - print("MD5 Checksum verified") + print("MD5 Checksum verified.") else: if ignore_corrupt == True: - print("WARNING: MD5 Checksum failed") + print("WARNING: MD5 Checksum failed!") else: - print("ERROR: MD5 Checksum failed") - print("set flag --ignorecorrupt to still use corrupted data") + print("ERROR: MD5 Checksum failed!") + print("MD5 from " + md5file + ":", md5_old) + print("MD5 from " + name + ".bpa: " + new_md5) + print("Set flag --ignorecorrupt to still use corrupted data.") exit() diff --git a/bigplanet/bp_get.py b/bigplanet/bp_get.py index ec2afe7..8587593 100755 --- a/bigplanet/bp_get.py +++ b/bigplanet/bp_get.py @@ -7,8 +7,8 @@ def DollarSign(m_bl, m_line, m_num, m_file): - '''Process each line looking for $ at the end, continue until no $ present''' - if m_line[-1] == '$': + """Process each line looking for $ at the end, continue until no $ present""" + if m_line[-1] == "$": m_bl.append(m_line[0:-1]) DollarSign(m_bl, m_file[m_num + 1], (m_num + 1), m_file) else: @@ -25,81 +25,98 @@ def ReadFile(bplSplitFile, verbose=False, archive=False): folder_name = "" SimName = "" - with open(bplSplitFile, 'r') as input: + with open(bplSplitFile, "r") as input: # first thing we check is if both include and exclude are in the file bc that is bad - if 'saKeyInclude' and 'saKeyExclude' in input.read(): - print("ERROR: saKeyInclude and saKeyExclude are mutually exclusive") + if "saKeyInclude" and "saKeyExclude" in input.read(): + print( + "ERROR: saKeyInclude and saKeyExclude are mutually exclusive" + ) exit() - with open(bplSplitFile, 'r') as input: + with open(bplSplitFile, "r") as input: # now we loop over the file and get the various inputs content = [line.strip().split() for line in input.readlines()] for num, line in enumerate(content): if line: # we get the folder where the raw data is stored and have the default output file name set - if line[0] == 'sDestFolder': + if line[0] == "sDestFolder": folder_name = line[1] - outputFile = folder_name.split('/')[-1] + "_filtered.bpf" - bpl_file = folder_name.split('/')[-1] + ".bpa" - if line[0] == 'sArchiveFile': + outputFile = folder_name.split("/")[-1] + "_filtered.bpf" + bpl_file = folder_name.split("/")[-1] + ".bpa" + if line[0] == "sArchiveFile": bpl_file = line[1] - if line[0] == 'sOutputFile': + if line[0] == "sOutputFile": outputFile = line[1] - if line[0] == 'sPrimaryFile': + if line[0] == "sPrimaryFile": primaryFile = line[1] - if line[0] == 'bUlysses': + if line[0] == "bUlysses": Ulysses = 1 - if line[0] == 'sSimName': + if line[0] == "sSimName": SimName = line[1] if line[0] == "saBodyFiles": DollarSign(bodylist, line[1:], num, content) bodylist = list(chain.from_iterable(bodylist)) - if line[0] == 'saKeyInclude': + if line[0] == "saKeyInclude": DollarSign(includelist, line[1:], num, content) includelist = list(chain.from_iterable(includelist)) - if line[0] == 'saKeyExclude': + if line[0] == "saKeyExclude": DollarSign(excludelist, line[1:], num, content) excludelist = list(chain.from_iterable(excludelist)) - if not bpl_file and not includelist and archive == False or not bpl_file and not excludelist and archive == False: - print("Error: No BPL Archive file or Include/Exclude List detected.") + if ( + not bpl_file + and not includelist + and archive == False + or not bpl_file + and not excludelist + and archive == False + ): + print( + "Error: No BPL Archive file or Include/Exclude List detected." + ) exit() - #print("WARNING: This may take some time...") + # print("WARNING: This may take some time...") if not folder_name: print("ERROR: No sDestFolder found in bpl.in file") exit() if Ulysses == 1: - outputFile = 'User.csv' - print("WARNING: Ulysses is set to True. Changed Output file to User.csv") + outputFile = "User.csv" + print( + "WARNING: Ulysses is set to True. Changed Output file to User.csv" + ) # if the simName is some value we need to check if all the keys are forward if SimName: for i in includelist: - if 'forward' not in i: + if "forward" not in i: print( - "ERROR: SimName must only use forward file data for output") + "ERROR: SimName must only use forward file data for output" + ) exit() for j in excludelist: - if 'forward' not in j: + if "forward" not in j: print( - "ERROR: SimName must only use forward file data for output") + "ERROR: SimName must only use forward file data for output" + ) exit() if Ulysses == True and SimName == "": for i in includelist: - if 'forward' in i: + if "forward" in i: print( - "ERROR: Forward keys in saKeyInclude requires sSimName for Ulysses Output") + "ERROR: Forward keys in saKeyInclude requires sSimName for Ulysses Output" + ) exit() for j in excludelist: - if 'forward' in j: + if "forward" in j: print( - "ERROR: Forward keys in saKeyExclude requires sSimName for Ulysses Output") + "ERROR: Forward keys in saKeyExclude requires sSimName for Ulysses Output" + ) exit() if verbose: @@ -114,40 +131,60 @@ def ReadFile(bplSplitFile, verbose=False, archive=False): print("Body Files:", bodylist) print("Primary File:", primaryFile) - return folder_name, bpl_file, outputFile, bodylist, primaryFile, includelist, excludelist, Ulysses, SimName + return ( + folder_name, + bpl_file, + outputFile, + bodylist, + primaryFile, + includelist, + excludelist, + Ulysses, + SimName, + ) def GetDir(vspace_file): - """ Give it input file and returns name of folder where simulations are located. """ + """Give it input file and returns name of folder where simulations are located.""" infiles = [] # gets the folder name with all the sims - with open(vspace_file, 'r') as vpl: + with open(vspace_file, "r") as vpl: content = [line.strip().split() for line in vpl.readlines()] for line in content: if line: - if line[0] == 'sDestFolder': + if line[0] == "sDestFolder": folder_name = line[1] - if line[0] == 'sBodyFile' or line[0] == 'sPrimaryFile': + if line[0] == "sBodyFile" or line[0] == "sPrimaryFile": infiles.append(line[1]) if folder_name is None: - raise IOError("Name of destination folder not provided in file '%s'." - "Use syntax 'destfolder '" % vspace_file) + raise IOError( + "Name of destination folder not provided in file '%s'." + "Use syntax 'destfolder '" % vspace_file + ) if os.path.isdir(folder_name) == False: - print("ERROR: Folder", folder_name, - "does not exist in the current directory.") + print( + "ERROR: Folder", + folder_name, + "does not exist in the current directory.", + ) exit() return folder_name, infiles def GetSims(folder_name, simname=""): - """ Pass it folder name where simulations are and returns list of simulation folders. """ + """Pass it folder name where simulations are and returns list of simulation folders.""" # gets the list of sims - sims = sorted([f.path for f in os.scandir( - os.path.abspath(folder_name)) if f.is_dir()]) + sims = sorted( + [ + f.path + for f in os.scandir(os.path.abspath(folder_name)) + if f.is_dir() + ] + ) if simname: sims = [x for x in sims if simname in x] @@ -165,19 +202,19 @@ def GetSNames(bodyfiles, sims): full_path = os.path.join(sims[0], file) # if the infile is the primary file, then get the system name if bodyfiles[-1] == file: - with open(full_path, 'r') as vpl: + with open(full_path, "r") as vpl: content = [line.strip().split() for line in vpl.readlines()] for line in content: if line: - if line[0] == 'sSystemName': + if line[0] == "sSystemName": system_name = line[1] # otherwise get the sName from the body files else: - with open(full_path, 'r') as infile: + with open(full_path, "r") as infile: content = [line.strip().split() for line in infile.readlines()] for line in content: if line: - if line[0] == 'sName': + if line[0] == "sName": body_names.append(line[1]) return system_name, body_names @@ -186,47 +223,67 @@ def GetSNames(bodyfiles, sims): def GetVplanetHelp(): command = "vplanet -H | egrep -v '^$|^\+' | cut -f 2,4 -d '|' | egrep '^ \*\*|^ Cust|^ Type|^ Dim|^ Defa|^Output Parameters'" py_ver = sys.version.split()[0] - if '3.6' in py_ver: - proc = sub.run(command, shell=True, universal_newlines=True, - stdout=sub.PIPE, stderr=sub.PIPE) + if "3.6" in py_ver: + proc = sub.run( + command, + shell=True, + universal_newlines=True, + stdout=sub.PIPE, + stderr=sub.PIPE, + ) else: - proc = sub.run(command, shell=True, text=True, - stdout=sub.PIPE, stderr=sub.PIPE) + proc = sub.run( + command, shell=True, text=True, stdout=sub.PIPE, stderr=sub.PIPE + ) output = proc.stdout.splitlines() vplanet_dict = {} for count, line in enumerate(output): - if ((line.startswith(' **b') or line.startswith(' **d') or line.startswith(' **i') or line.startswith(' **s')) or line.startswith(' **sa') and len(line.split()) == 1): + if ( + ( + line.startswith(" **b") + or line.startswith(" **d") + or line.startswith(" **i") + or line.startswith(" **s") + ) + or line.startswith(" **sa") + and len(line.split()) == 1 + ): option = line.strip("** ") - # print("Option:",option) + #print("Option: ",option,repr(count)) vplanet_dict[option] = {} num = count + 1 while num != count: + #print("num,count: ",repr(num),repr(len(output))) + #print("Output[num]: ",repr(output[num])) + if (num >= len(output)): + print("ERROR: Unable to gather VPLanet help information!") + assert(0) if "Type" in output[num]: - tp = output[num].rpartition('|')[-1].strip() - vplanet_dict[option]['Type'] = tp - # print("Type:",tp) + tp = output[num].rpartition("|")[-1].strip() + vplanet_dict[option]["Type"] = tp + #print("Type:",tp) num += 1 elif "Custom unit" in output[num]: - custom_unit = output[num].rpartition('|')[-1].strip() - vplanet_dict[option]['Custom Unit'] = custom_unit + custom_unit = output[num].rpartition("|")[-1].strip() + vplanet_dict[option]["Custom Unit"] = custom_unit #print("Custom Unit:",custom_unit) num += 1 elif "Dimension(s)" in output[num]: - dim = output[num].rpartition('|')[-1].strip() - vplanet_dict[option]['Dimension'] = dim - # print("Dimension:",dim) + dim = output[num].rpartition("|")[-1].strip() + vplanet_dict[option]["Dimension"] = dim + #print("Dimension:",dim) num += 1 elif "Default value" in output[num]: - default = output[num].rpartition('|')[-1].strip() - vplanet_dict[option]['Default Value'] = default + default = output[num].rpartition("|")[-1].strip() + vplanet_dict[option]["Default Value"] = default #print("Default Value",default) - # print() + #print() num += 1 @@ -247,11 +304,11 @@ def GetLogName(in_files, sims, system_name): for file in in_files: full_path = os.path.join(sims[0], file) - with open(full_path, 'r') as vpl: + with open(full_path, "r") as vpl: content = [line.strip().split() for line in vpl.readlines()] for line in content: if line: - if line[0] == 'sLogfile': + if line[0] == "sLogfile": prefix = line[1] logfile = prefix + ".log" diff --git a/bigplanet/bp_process.py b/bigplanet/bp_process.py index 6c8c919..ab497aa 100755 --- a/bigplanet/bp_process.py +++ b/bigplanet/bp_process.py @@ -1,18 +1,19 @@ #!/usr/bin/env python import os -import pandas as pd -import numpy as np + import h5py +import numpy as np +import pandas as pd def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): - prop = '' - body = 'system' + prop = "" + body = "system" path = os.path.join(folder, logfile) if verbose == True: print(path) - with open(path, 'r+', errors='ignore') as log: + with open(path, "r+", errors="ignore") as log: content = [line.strip() for line in log.readlines()] for line in content: @@ -21,32 +22,36 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): continue # First we need to get the body names and if its a inital or final value - if line.startswith('-'): - tmp_line = line.replace( - '-', '').replace(':', '').strip().replace(' ', '_') + if line.startswith("-"): + tmp_line = ( + line.replace("-", "") + .replace(":", "") + .strip() + .replace(" ", "_") + ) - if tmp_line.startswith('INITIAL_SYSTEM_PROPERTIES'): - prop = 'initial' + if tmp_line.startswith("INITIAL_SYSTEM_PROPERTIES"): + prop = "initial" - if tmp_line.startswith('FINAL_SYSTEM_PROPERTIES'): - prop = 'final' - body = 'system' + if tmp_line.startswith("FINAL_SYSTEM_PROPERTIES"): + prop = "final" + body = "system" - if tmp_line.startswith('BODY'): - body = tmp_line[tmp_line.find('_')+1:].strip() + if tmp_line.startswith("BODY"): + body = tmp_line[tmp_line.find("_") + 1 :].strip() continue # if the line starts with a '(' that means its a variable we need to grab the units - if line.startswith('('): - fv_param = line[1:line.find(')')].strip() - units = line[line.find('[')+1:line.find(']')].strip() + if line.startswith("("): + fv_param = line[1 : line.find(")")].strip() + units = line[line.rfind("[") + 1 : line.rfind("]")].strip() if not units: - units = 'nd' + units = "nd" - fv_value = line[line.find(':')+1:].strip() - key_name = body + ':' + fv_param + ':' + prop + fv_value = line[line.find(":") + 1 :].strip() + key_name = body + ":" + fv_param + ":" + prop # make this into a function if incl is not None: @@ -66,25 +71,25 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): data[key_name] = [units, fv_value] # if the name starts with output order then its a list of variables - if line.startswith('Output Order') and len(line[line.find(':'):]) > 1: - parm_key = line[:line.find(':')].replace(' ', '') - params = line[line.find(':') + 1:].strip().split(']') - key_name = body + ':' + parm_key + if line.startswith("Output Order") and len(line[line.find(":") :]) > 1: + parm_key = line[: line.find(":")].replace(" ", "") + params = line[line.find(":") + 1 :].strip().split("]") + key_name = body + ":" + parm_key out_params = [] for i in params: - var = i[:i.find('[')].strip() - units = i[i.find('[') + 1:] + var = i[: i.find("[")].strip() + units = i[i.find("[") + 1 :] if not units: - units = 'nd' + units = "nd" - if var == '': + if var == "": continue out_params.append([var, units]) - key_name_forward = body + ':' + var + ':forward' + key_name_forward = body + ":" + var + ":forward" # make this into a function if incl is not None: @@ -112,25 +117,28 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): data[key_name] = out_params # if the name starts with grid output order then its a list of variables - if line.startswith('Grid Output Order') and len(line[line.find(':'):]) > 1: - parm_key = line[:line.find(':')].replace(' ', '') - params = line[line.find(':') + 1:].strip().split(']') - key_name = body + ':' + parm_key + if ( + line.startswith("Grid Output Order") + and len(line[line.find(":") :]) > 1 + ): + parm_key = line[: line.find(":")].replace(" ", "") + params = line[line.find(":") + 1 :].strip().split("]") + key_name = body + ":" + parm_key out_params = [] for i in params: - var = i[:i.find('[')].strip() - units = i[i.find('[') + 1:] + var = i[: i.find("[")].strip() + units = i[i.find("[") + 1 :] if not units: - units = 'nd' + units = "nd" - if var == '': + if var == "": continue out_params.append([var, units]) - key_name_climate = body + ':' + var + ':climate' + key_name_climate = body + ":" + var + ":climate" # make this into a function if incl is not None: @@ -158,7 +166,9 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): return data -def ProcessOutputfile(file, data, body, Output, prefix, folder, verbose, incl=None, excl=None): +def ProcessOutputfile( + file, data, body, Output, prefix, folder, verbose, incl=None, excl=None +): path = os.path.join(folder, file) if verbose == True: @@ -169,8 +179,8 @@ def ProcessOutputfile(file, data, body, Output, prefix, folder, verbose, incl=No for k, v in Output.items(): for num in v: header.append(num[0]) - if num[1] == '': - units.append('nd') + if num[1] == "": + units.append("nd") else: units.append(num[1]) @@ -178,7 +188,7 @@ def ProcessOutputfile(file, data, body, Output, prefix, folder, verbose, incl=No sorted = sorted.to_numpy().transpose().tolist() for i, row in enumerate(sorted): - key_name = body + ':' + header[i] + prefix + key_name = body + ":" + header[i] + prefix # if key_name in data: # data[key_name].append(row) @@ -206,8 +216,10 @@ def ProcessOutputfile(file, data, body, Output, prefix, folder, verbose, incl=No return data -def ProcessSeasonalClimatefile(prefix, data, body, name, folder, verbose, incl=None, excl=None): - file_name = prefix + '.' + name + '.0' +def ProcessSeasonalClimatefile( + prefix, data, body, name, folder, verbose, incl=None, excl=None +): + file_name = prefix + "." + name + ".0" path = os.path.join(folder, "SeasonalClimateFiles/", file_name) if verbose == True: @@ -216,19 +228,23 @@ def ProcessSeasonalClimatefile(prefix, data, body, name, folder, verbose, incl=N sorted = pd.read_csv(path, header=None, delim_whitespace=True).to_numpy() sorted = sorted.transpose().tolist() - key_name = body + ':' + name - units = '' - if (name == 'DailyInsol' or name == 'SeasonalFIn' or - name == 'SeasonalFOut' or name == 'SeasonalDivF'): - units = 'W/m^2' - if name == 'PlanckB': - units = 'W/m^2/K' - if name == 'SeasonalIceBalance': - units = 'kg/m^2/s' - if name == 'SeasonalTemp': - units = 'deg C' - if name == 'SeasonalFMerid': - units = 'W' + key_name = body + ":" + name + units = "" + if ( + name == "DailyInsol" + or name == "SeasonalFIn" + or name == "SeasonalFOut" + or name == "SeasonalDivF" + ): + units = "W/m^2" + if name == "PlanckB": + units = "W/m^2/K" + if name == "SeasonalIceBalance": + units = "kg/m^2/s" + if name == "SeasonalTemp": + units = "deg C" + if name == "SeasonalFMerid": + units = "W" # if key_name not in data: # data[key_name]= [units, sorted] @@ -254,10 +270,12 @@ def ProcessSeasonalClimatefile(prefix, data, body, name, folder, verbose, incl=N return data -def ProcessInputfile(data, in_file, folder, vplanet_help, verbose, incl=None, excl=None): +def ProcessInputfile( + data, in_file, folder, vplanet_help, verbose, incl=None, excl=None +): # set the body name equal to the infile name - body = in_file.partition('.')[0] + body = in_file.partition(".")[0] path = os.path.join(folder, in_file) if verbose == True: print(path) @@ -269,19 +287,19 @@ def ProcessInputfile(data, in_file, folder, vplanet_help, verbose, incl=None, ex # for every line in the array check if the line is blank # or if the line starts with a # next = False - t_line = '' + t_line = "" for num, line in enumerate(content): - if len(line) == 0 or line.startswith('#'): + if len(line) == 0 or line.startswith("#"): continue # if theres a comment in the line we don't want that, so partition the # string and use everything before it - if '#' in line: - line = line.partition('#')[0] + if "#" in line: + line = line.partition("#")[0] # if there's a $ we need to get the next line and append it - if '$' in line: + if "$" in line: next = True - line = line.partition('$')[0] + line = line.partition("$")[0] t_line = t_line + line continue @@ -289,20 +307,20 @@ def ProcessInputfile(data, in_file, folder, vplanet_help, verbose, incl=None, ex next = False t_line = t_line + line line = t_line - t_line = '' + t_line = "" line = line.split() key = line[0] value = line[1] - key = key.replace('-', '') - key_name = body + ':' + key + ':option' + key = key.replace("-", "") + key_name = body + ":" + key + ":option" units = ProcessInfileUnits(key, value, folder, path, vplanet_help) - if 'saOutputOrder' in key_name or 'saGridOutput' in key_name: + if "saOutputOrder" in key_name or "saGridOutput" in key_name: for i in value: - if value[0] == '-': + if value[0] == "-": value = value[1:] # make this into a function @@ -327,78 +345,95 @@ def ProcessInputfile(data, in_file, folder, vplanet_help, verbose, incl=None, ex def ProcessInfileUnits(name, value, folder, in_file, vplanet_help): # check if the value is negative and has a negative option - custom_unit = vplanet_help.get(name, {}).get('Custom Units') - if '-' in value and custom_unit != None: + custom_unit = vplanet_help.get(name, {}).get("Custom Units") + if "-" in value and custom_unit != None: unit = custom_unit return unit else: - dim = vplanet_help.get(name, {}).get('Dimension') + dim = vplanet_help.get(name, {}).get("Dimension") # since pressure and energy arent sUnits, we have to replace them with the dims for each - if dim == None or dim == 'nd': - unit = 'nd' + if dim == None or dim == "nd": + unit = "nd" return unit - if 'pressure' in dim: - dim = dim.replace('pressure', '(mass*length^-1*time^-2)') - if 'energy' in dim: - dim = dim.replace('energy', '(mass*length^2*time^-2)') + if "pressure" in dim: + dim = dim.replace("pressure", "(mass*length^-1*time^-2)") + if "energy" in dim: + dim = dim.replace("energy", "(mass*length^2*time^-2)") # check the options file the value was in and see if the Dimension is there - with open(in_file, 'r+') as infile: + with open(in_file, "r+") as infile: infile_lines = infile.readlines() for infile_line in infile_lines: - if 'sUnitLength' in infile_line and 'length' in dim: - dim = dim.replace('length', infile_line.split()[1]) - if 'sUnitAngle' in infile_line and 'angle' in dim: - dim = dim.replace('angle', infile_line.split()[1]) - if 'sUnitTemp' in infile_line and 'temperature' in dim: - dim = dim.replace('temperature', infile_line.split()[1]) - if 'sUnitMass' in infile_line and 'mass' in dim: - dim = dim.replace('mass', infile_line.split()[1]) - if 'sUnitTime' in infile_line and 'time' in dim: - dim = dim.replace('time', infile_line.split()[1]) + if "sUnitLength" in infile_line and "length" in dim: + dim = dim.replace("length", infile_line.split()[1]) + if "sUnitAngle" in infile_line and "angle" in dim: + dim = dim.replace("angle", infile_line.split()[1]) + if "sUnitTemp" in infile_line and "temperature" in dim: + dim = dim.replace("temperature", infile_line.split()[1]) + if "sUnitMass" in infile_line and "mass" in dim: + dim = dim.replace("mass", infile_line.split()[1]) + if "sUnitTime" in infile_line and "time" in dim: + dim = dim.replace("time", infile_line.split()[1]) if infile_line == infile_lines[-1]: # if its not in the options file, it might be in in the vpl.in file - with open(os.path.join(folder, 'vpl.in'), 'r+') as vplfile: + with open(os.path.join(folder, "vpl.in"), "r+") as vplfile: vpl_lines = vplfile.readlines() for vpl_line in vpl_lines: - if 'sUnitLength' in vpl_line and 'length' in dim: - dim = dim.replace('length', vpl_line.split()[1]) - if 'sUnitAngle' in vpl_line and 'angle' in dim: - dim = dim.replace('angle', vpl_line.split()[1]) - if 'sUnitTemp' in vpl_line and 'temperature' in dim: - dim = dim.replace('temperature', vpl_line.split()[1]) - if 'sUnitMass' in vpl_line and 'mass' in dim: - dim = dim.replace('mass', vpl_line.split()[1]) - if 'sUnitTime' in vpl_line and 'time' in dim: - dim = dim.replace('time', vpl_line.split()[1]) + if "sUnitLength" in vpl_line and "length" in dim: + dim = dim.replace("length", vpl_line.split()[1]) + if "sUnitAngle" in vpl_line and "angle" in dim: + dim = dim.replace("angle", vpl_line.split()[1]) + if "sUnitTemp" in vpl_line and "temperature" in dim: + dim = dim.replace("temperature", vpl_line.split()[1]) + if "sUnitMass" in vpl_line and "mass" in dim: + dim = dim.replace("mass", vpl_line.split()[1]) + if "sUnitTime" in vpl_line and "time" in dim: + dim = dim.replace("time", vpl_line.split()[1]) # the only place left is the default of sUnit of the Dimension if vpl_line == vpl_lines[-1]: - if 'length' in dim: + if "length" in dim: dim = dim.replace( - 'length', vplanet_help['sUnitLength']['Default Value']) - if 'angle' in dim: + "length", + vplanet_help["sUnitLength"]["Default Value"], + ) + if "angle" in dim: dim = dim.replace( - 'angle', vplanet_help['sUnitAngle']['Default Value']) - if 'temperature' in dim: + "angle", + vplanet_help["sUnitAngle"]["Default Value"], + ) + if "temperature" in dim: dim = dim.replace( - 'temperature', vplanet_help['sUnitTemp']['Default Value']) - if 'mass' in dim: + "temperature", + vplanet_help["sUnitTemp"]["Default Value"], + ) + if "mass" in dim: dim = dim.replace( - 'mass', vplanet_help['sUnitMass']['Default Value']) - if 'time' in dim: + "mass", vplanet_help["sUnitMass"]["Default Value"] + ) + if "time" in dim: dim = dim.replace( - 'time', vplanet_help['sUnitTime']['Default Value']) + "time", vplanet_help["sUnitTime"]["Default Value"] + ) unit = dim return unit -def GatherData(data, system_name, body_names, logfile, in_files, vplanet_help, folder, verbose): +def GatherData( + data, + system_name, + body_names, + logfile, + in_files, + vplanet_help, + folder, + verbose, +): """ .... """ @@ -417,7 +452,7 @@ def GatherData(data, system_name, body_names, logfile, in_files, vplanet_help, f gridoutputorder = body + ":GridOutputOrder" # if output order from the log file isn't empty process it if outputorder in data: - #OutputOrder = data[outputorder] + # OutputOrder = data[outputorder] OutputOrder = {} OutputOrder[outputorder] = data[outputorder] @@ -426,48 +461,67 @@ def GatherData(data, system_name, body_names, logfile, in_files, vplanet_help, f file_name = data[Outfile] else: # need to figure out if its forward file or backwards file - forwardOption = in_files[-1].partition( - '.')[0] + ":bDoForward:option" - backwardOption = in_files[-1].partition( - '.')[0] + ":bDoBackward:option" + forwardOption = ( + in_files[-1].partition(".")[0] + ":bDoForward:option" + ) + backwardOption = ( + in_files[-1].partition(".")[0] + ":bDoBackward:option" + ) if forwardOption in data: - file_name = system_name + '.' + body + '.forward' + file_name = system_name + "." + body + ".forward" prefix = ":forward" elif backwardOption in data: - file_name = system_name + '.' + body + '.backward' + file_name = system_name + "." + body + ".backward" prefix = ":backward" data = ProcessOutputfile( - file_name, data, body, OutputOrder, prefix, folder, verbose) + file_name, data, body, OutputOrder, prefix, folder, verbose + ) # now process the grid output order (if it exists) if gridoutputorder in data: GridOutputOrder = {} GridOutputOrder[gridoutputorder] = data[gridoutputorder] - climate_name = system_name + '.' + body + '.Climate' + climate_name = system_name + "." + body + ".Climate" data = ProcessOutputfile( - climate_name, data, body, GridOutputOrder, ':climate', folder, verbose) - prefix = system_name + '.' + body - name = ['DailyInsol', 'PlanckB', 'SeasonalDivF', 'SeasonalFIn', - 'SeasonalFMerid', 'SeasonalFOut', 'SeasonalIceBalance', - 'SeasonalTemp'] + climate_name, + data, + body, + GridOutputOrder, + ":climate", + folder, + verbose, + ) + prefix = system_name + "." + body + name = [ + "DailyInsol", + "PlanckB", + "SeasonalDivF", + "SeasonalFIn", + "SeasonalFMerid", + "SeasonalFOut", + "SeasonalIceBalance", + "SeasonalTemp", + ] for i in range(len(name)): data = ProcessSeasonalClimatefile( - prefix, data, body, name[i], folder, verbose) + prefix, data, body, name[i], folder, verbose + ) return data -def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive=True): +def DictToBP( + data, vplanet_help, h5_file, verbose=False, group_name="", archive=True +): for k, v in data.items(): var = k.split(":")[1] - end = k.split(':')[-1] if "OutputOrder" in var or "GridOutput" in var: v_value = v - v_attr = '' + v_attr = "" else: v_attr = v[0] @@ -480,13 +534,16 @@ def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive= tp = "float" else: - if vplanet_help.get(var, {}).get('Type') == 'String' or vplanet_help.get(var, {}).get('Type') == 'String-Array': + if ( + vplanet_help.get(var, {}).get("Type") == "String" + or vplanet_help.get(var, {}).get("Type") == "String-Array" + ): tp = "S" else: tp = "float" if archive == True and group_name: - dataset_name = group_name + '/' + k + dataset_name = group_name + "/" + k else: dataset_name = k @@ -498,6 +555,6 @@ def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive= print("Value:", v_value) print() - h5_file.create_dataset(dataset_name, data=v_value) + h5_file.create_dataset(dataset_name, data=np.array([v_value], dtype=tp)) - h5_file[dataset_name].attrs['Units'] = v_attr + h5_file[dataset_name].attrs["Units"] = v_attr diff --git a/bigplanet/bpstatus.py b/bigplanet/bpstatus.py index c648270..4129450 100644 --- a/bigplanet/bpstatus.py +++ b/bigplanet/bpstatus.py @@ -1,46 +1,49 @@ +import argparse import os import sys -import argparse def bpstatus(input_file): - with open(input_file, 'r') as vsf: + with open(input_file, "r") as vsf: vspace_all = vsf.readlines() dest_line = vspace_all[1] folder_name = dest_line.strip().split(None, 1)[1] if folder_name is None: raise IOError( - "Name of destination folder not provided in file '%s'. Use syntax 'destfolder '" % input_file) + "Name of destination folder not provided in file '%s'. Use syntax 'destfolder '" + % input_file + ) count_done = 0 count_todo = 0 count_ip = 0 - checkpoint_file = os.getcwd() + '/.' + folder_name + '_BPL' + checkpoint_file = os.getcwd() + "/." + folder_name + "_BPL" if os.path.isfile(checkpoint_file) == False: raise Exception("BigPlanet must be running prior to using bpstatus") else: - with open(checkpoint_file, 'r') as cp: + with open(checkpoint_file, "r") as cp: content = [line.strip().split() for line in cp.readlines()] for number, line in enumerate(content): status = line[1] - if status == '1': + if status == "1": count_done += 1 - elif status == '-1': + elif status == "-1": count_todo += 1 - elif status == '0': + elif status == "0": count_ip += 1 - print('--BigPlanet Status--') - print('Number of Simulations completed: ' + str(count_done)) - print('Number of Simulations in progress: ' + str(count_ip)) - print('Number of Simulations remaining: ' + str(count_todo)) + print("--BigPlanet Status--") + print("Number of Simulations completed: " + str(count_done)) + print("Number of Simulations in progress: " + str(count_ip)) + print("Number of Simulations remaining: " + str(count_todo)) def main(): parser = argparse.ArgumentParser( - description="Check Status of BigPlanet Data Extraction") + description="Check Status of BigPlanet Data Extraction" + ) parser.add_argument("InputFile", help="Name of the vspace input file") args = parser.parse_args() diff --git a/environment.yml b/environment.yml index 7d184a2..6b8a316 100644 --- a/environment.yml +++ b/environment.yml @@ -2,22 +2,22 @@ name: vplanet channels: - defaults dependencies: - - numpy==1.19.2 - - pip==20.3.3 - - scipy==1.5.2 - - setuptools==51.3.3 + - numpy>=1.19.2 + - pip>=20.3.3 + - scipy>=1.5.2 + - setuptools>=51.3.3 - pip: - - matplotlib==3.3.3 - - nose==1.3.7 - - parameterized==0.8.1 - - pytest==6.2.4 - - pytest-dependency==0.5.1 - - pytest-env==0.6.2 - - pytest-cov==2.11.1 - - sphinx==4.2.0 - - breathe==4.30.0 - - sphinxcontrib-bibtex==2.2.0 - - coverage==5.5 - - argparse==1.4 - - h5py==3.1 - - sphinx_rtd_theme==1.0.0 + - matplotlib>=3.3.3 + - nose>=1.3.7 + - parameterized==0.8.1 + - pytest>=6.2.4 + - pytest-dependency>=0.5.1 + - pytest-env>=0.6.2 + - pytest-cov>=2.11.1 + - sphinx==4.2.0 + - breathe>=4.30.0 + - sphinxcontrib-bibtex>=2.2.0 + - coverage>=5.5 + - argparse>=1.4 + - h5py>=3.1 + - sphinx_rtd_theme==1.0.0 diff --git a/setup.py b/setup.py index 9afd061..86686c8 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from setuptools import setup import os +from setuptools import setup # Setup! setup( @@ -20,8 +20,7 @@ "write_to": os.path.join("bigplanet", "bigplanet_version.py"), "write_to_template": '__version__ = "{version}"\n', }, - install_requires=["numpy", "h5py", "argparse", - "scipy", "pandas"], + install_requires=["numpy", "h5py", "argparse", "scipy", "pandas"], entry_points={ "console_scripts": [ "bigplanet = bigplanet.bigplanet:Arguments", diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..38a3510 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,2 @@ +.* +BP_* \ No newline at end of file diff --git a/tests/CreateHDF5/test_CreateHDF5.py b/tests/CreateHDF5/test_CreateHDF5.py new file mode 100644 index 0000000..4589c7e --- /dev/null +++ b/tests/CreateHDF5/test_CreateHDF5.py @@ -0,0 +1,56 @@ +import multiprocessing as mp +import os +import pathlib +import subprocess +import sys +import warnings +import shutil + +import numpy as np + + +def test_CreateHDF5(): + # gets current path + path = pathlib.Path(__file__).parents[0].absolute() + sys.path.insert(1, str(path.parents[0])) + + # gets the number of cores on the machine + cores = mp.cpu_count() + if cores == 1: + warnings.warn("There is only 1 core on the machine", stacklevel=3) + else: + # Remove anything from previous tests + if (path / "BP_CreateHDF5").exists(): + shutil.rmtree(path / "BP_CreateHDF5") + if (path / ".BP_CreateHDF5").exists(): + os.remove(path / ".BP_CreateHDF5") + if (path / ".BP_CreateHDF5_BPL").exists(): + os.remove(path / ".BP_CreateHDF5_BPL") + if (path / "BP_CreateHDF5.bpa").exists(): + os.remove(path / "BP_CreateHDF5.bpa") + if (path / "BP_CreateHDF5.md5").exists(): + os.remove(path / "BP_CreateHDF5.md5") + + # Run vspace + print("Running vspace.") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multi-planet + print("Running MultiPlanet.") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run bigplanet + print("Running BigPlanet.") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + + file = path / "BP_CreateHDF5.bpa" + + # checks if the bpl files exist + assert os.path.isfile(file) == True + + +if __name__ == "__main__": + test_CreateHDF5() diff --git a/tests/CreateHDF5/test_bpcreatehdf5.py b/tests/CreateHDF5/test_bpcreatehdf5.py deleted file mode 100644 index c46b399..0000000 --- a/tests/CreateHDF5/test_bpcreatehdf5.py +++ /dev/null @@ -1,39 +0,0 @@ -import subprocess -import numpy as np -import os -import pathlib -import multiprocessing as mp -import warnings -import sys - - -def test_bpcreatehdf5(): - # gets current path - path = pathlib.Path(__file__).parents[0].absolute() - sys.path.insert(1, str(path.parents[0])) - - # gets the number of cores on the machine - cores = mp.cpu_count() - if cores == 1: - warnings.warn("There is only 1 core on the machine", stacklevel=3) - else: - # Run vspace - if not (path / "BP_CreateHDF5").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_CreateHDF5").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / ".BP_CreateHDF5_BPL").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) - - file = (path / "BP_CreateHDF5.bpa") - - # checks if the bpl files exist - assert os.path.isfile(file) == True - - -if __name__ == "__main__": - test_bpcreatehdf5() diff --git a/tests/ExtractArchive/test_ExtractArchive.py b/tests/ExtractArchive/test_ExtractArchive.py new file mode 100644 index 0000000..b03927d --- /dev/null +++ b/tests/ExtractArchive/test_ExtractArchive.py @@ -0,0 +1,65 @@ +import multiprocessing as mp +import os +import pathlib +import subprocess +import sys +import warnings +import shutil +import numpy as np +import bigplanet as bp + + +def test_ExtractArchive(): + # gets current path + path = pathlib.Path(__file__).parents[0].absolute() + sys.path.insert(1, str(path.parents[0])) + + # gets the number of cores on the machine + cores = mp.cpu_count() + if cores == 1: + warnings.warn("There is only 1 core on the machine", stacklevel=3) + else: + # Remove anything from previous tests + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / ".BP_Extract_BPL").exists(): + os.remove(path / ".BP_Extract_BPL") + if (path / "BP_Extract.bpa").exists(): + os.remove(path / "BP_Extract.bpa") + if (path / "../BP_Extract.md5").exists(): + os.remove(path / "../BP_Extract.md5") + if (path / "BP_Extract.md5").exists(): + os.remove(path / "BP_Extract.md5") + + # Run vspace + print("Running vspace.") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multi-planet + print("Running MultiPlanet.") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run bigplanet + print("Running BigPlanet.") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + + file = bp.BPLFile(path / "BP_Extract.bpa") + + earth_Instellation_final = bp.ExtractColumn( + file, "earth:Instellation:final" + ) + sun_RotPer_initial = bp.ExtractColumn(file, "sun:RotPer:initial") + + assert np.isclose(earth_Instellation_final[0], 1367.635318) + assert np.isclose(earth_Instellation_final[1], 341.90883) + + assert np.isclose(sun_RotPer_initial[0], 86400.0) + + +if __name__ == "__main__": + test_ExtractArchive() diff --git a/tests/ExtractArchive/test_bpextract.py b/tests/ExtractArchive/test_bpextract.py deleted file mode 100644 index 8b04952..0000000 --- a/tests/ExtractArchive/test_bpextract.py +++ /dev/null @@ -1,47 +0,0 @@ -import subprocess -import numpy as np -import os -import pathlib -import warnings -import h5py -import multiprocessing as mp -import sys -import bigplanet as bp - - -def test_bpextract(): - # gets current path - path = pathlib.Path(__file__).parents[0].absolute() - sys.path.insert(1, str(path.parents[0])) - - # gets the number of cores on the machine - cores = mp.cpu_count() - if cores == 1: - warnings.warn("There is only 1 core on the machine", stacklevel=3) - else: - # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / ".BP_Extract_BPL").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) - - file = bp.BPLFile(path / "BP_Extract.bpa") - - earth_Instellation_final = bp.ExtractColumn( - file, 'earth:Instellation:final') - sun_RotPer_initial = bp.ExtractColumn(file, 'sun:RotPer:initial') - - assert np.isclose(earth_Instellation_final[0], 1367.635318) - assert np.isclose(earth_Instellation_final[1], 341.90883) - - assert np.isclose(sun_RotPer_initial[0], 86400.0) - - -if __name__ == "__main__": - test_bpextract() diff --git a/tests/ExtractFilterArchive/test_ExtractFilterArchive.py b/tests/ExtractFilterArchive/test_ExtractFilterArchive.py new file mode 100644 index 0000000..bd390f4 --- /dev/null +++ b/tests/ExtractFilterArchive/test_ExtractFilterArchive.py @@ -0,0 +1,78 @@ +import multiprocessing as mp +import os +import pathlib +import subprocess +import sys +import warnings +import shutil +import numpy as np +import bigplanet as bp + + +def test_ExtractFilterArchive(): + # gets current path + path = pathlib.Path(__file__).parents[0].absolute() + sys.path.insert(1, str(path.parents[0])) + + # gets the number of cores on the machine + cores = mp.cpu_count() + if cores == 1: + warnings.warn("There is only 1 core on the machine", stacklevel=3) + else: + # Remove any old test files/dirs + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / ".BP_Extract_BPL").exists(): + os.remove(path / ".BP_Extract_BPL") + if (path / "BP_Extract.bpa").exists(): + os.remove(path / "BP_Extract.bpa") + if (path / "Test.bpf").exists(): + os.remove(path / "Test.bpf") + if (path / "../BP_Extract.md5").exists(): + os.remove(path / "../BP_Extract.md5") + if (path / "BP_Extract.md5").exists(): + os.remove(path / "BP_Extract.md5") + + # Run vspace + print("Running vspace.") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multi-planet + print("Running MultiPlanet.") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run bigplanet + print("Creating BigPlanet archive.") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + + # Run bigplanet + print("Creating BigPlanet file.") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + + file = bp.BPLFile(path / "Test.bpf") + + earth_Instellation_final = bp.ExtractColumn( + file, "earth:Instellation:final" + ) + sun_Luminosity_option = bp.ExtractColumn( + file, "sun:dLuminosity:option" + ) + earth_Mass_option = bp.ExtractColumn(file, "earth:dMass:option") + vpl_stoptime_option = bp.ExtractColumn(file, "vpl:dStopTime:option") + earth_tman_forward = bp.ExtractColumn(file, "earth:TMan:forward") + + assert np.isclose(earth_Instellation_final[1], 341.90883) + assert np.isclose(sun_Luminosity_option[0], 3.846e26) + assert np.isclose(earth_Mass_option[1], -1.5) + assert np.isclose(vpl_stoptime_option[0], 4.5e9) + assert np.isclose(earth_tman_forward[0][0], 3000.0) + + +if __name__ == "__main__": + test_ExtractFilterArchive() diff --git a/tests/ExtractFilterArchive/test_bpextractfilterarchive.py b/tests/ExtractFilterArchive/test_bpextractfilterarchive.py deleted file mode 100644 index 5c027e6..0000000 --- a/tests/ExtractFilterArchive/test_bpextractfilterarchive.py +++ /dev/null @@ -1,56 +0,0 @@ -import subprocess -import numpy as np -import os -import pathlib -import warnings -import h5py -import multiprocessing as mp -import sys -import bigplanet as bp - - -def test_bpextract(): - # gets current path - path = pathlib.Path(__file__).parents[0].absolute() - sys.path.insert(1, str(path.parents[0])) - - # gets the number of cores on the machine - cores = mp.cpu_count() - if cores == 1: - warnings.warn("There is only 1 core on the machine", stacklevel=3) - else: - # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / "BP_Extract.bpa").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) - - # Run bigplanet - if not (path / "Test.bpf").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) - - file = bp.BPLFile(path / "Test.bpf") - - earth_Instellation_final = bp.ExtractColumn( - file, 'earth:Instellation:final') - sun_Luminosity_option = bp.ExtractColumn( - file, 'sun:dLuminosity:option') - earth_Mass_option = bp.ExtractColumn(file, 'earth:dMass:option') - vpl_stoptime_option = bp.ExtractColumn(file, 'vpl:dStopTime:option') - earth_tman_forward = bp.ExtractColumn(file, 'earth:TMan:forward') - - assert np.isclose(earth_Instellation_final[1], 341.90883) - assert np.isclose(sun_Luminosity_option[0], 3.846e26) - assert np.isclose(earth_Mass_option[1], -1.5) - assert np.isclose(vpl_stoptime_option[0], 4.5e9) - assert np.isclose(earth_tman_forward[0][0], 3000.0) - - -if __name__ == "__main__": - test_bpextract() diff --git a/tests/Bpstatus/__init__.py b/tests/ExtractFilterRaw/__init__.py similarity index 100% rename from tests/Bpstatus/__init__.py rename to tests/ExtractFilterRaw/__init__.py diff --git a/tests/ExtractFilterRawData/bpl.in b/tests/ExtractFilterRaw/bpl.in similarity index 100% rename from tests/ExtractFilterRawData/bpl.in rename to tests/ExtractFilterRaw/bpl.in diff --git a/tests/Bpstatus/earth.in b/tests/ExtractFilterRaw/earth.in similarity index 100% rename from tests/Bpstatus/earth.in rename to tests/ExtractFilterRaw/earth.in diff --git a/tests/Bpstatus/sun.in b/tests/ExtractFilterRaw/sun.in similarity index 100% rename from tests/Bpstatus/sun.in rename to tests/ExtractFilterRaw/sun.in diff --git a/tests/ExtractFilterRaw/test_ExtractFilterRaw.py b/tests/ExtractFilterRaw/test_ExtractFilterRaw.py new file mode 100644 index 0000000..9471ab2 --- /dev/null +++ b/tests/ExtractFilterRaw/test_ExtractFilterRaw.py @@ -0,0 +1,68 @@ +import multiprocessing as mp +import os +import pathlib +import subprocess +import sys +import warnings +import shutil +import h5py +import numpy as np + +import bigplanet as bp + + +def test_ExtractFilterRaw(): + # gets current path + path = pathlib.Path(__file__).parents[0].absolute() + sys.path.insert(1, str(path.parents[0])) + + # gets the number of cores on the machine + cores = mp.cpu_count() + if cores == 1: + warnings.warn("There is only 1 core on the machine", stacklevel=3) + else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / "Test.bpf").exists(): + os.remove(path / "Test.bpf") + + # Run vspace + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multiplanet + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run BigPlanet + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + + file = bp.BPLFile(path / "Test.bpf") + + earth_Instellation_final = bp.ExtractColumn( + file, "earth:Instellation:final" + ) + sun_Luminosity_option = bp.ExtractColumn( + file, "sun:dLuminosity:option" + ) + earth_Mass_option = bp.ExtractColumn(file, "earth:dMass:option") + vpl_stoptime_option = bp.ExtractColumn(file, "vpl:dStopTime:option") + earth_tman_forward = bp.ExtractColumn(file, "earth:TMan:forward") + + print(earth_Instellation_final[1]) + assert np.isclose(earth_Instellation_final[1], 341.90883) + assert np.isclose(sun_Luminosity_option[0], 3.846e26) + assert np.isclose(earth_Mass_option[1], -1.5) + assert np.isclose(vpl_stoptime_option[0], 4.5e9) + assert np.isclose(earth_tman_forward[0][0], 3000.0) + + +if __name__ == "__main__": + test_ExtractFilterRaw() diff --git a/tests/Bpstatus/vpl.in b/tests/ExtractFilterRaw/vpl.in similarity index 100% rename from tests/Bpstatus/vpl.in rename to tests/ExtractFilterRaw/vpl.in diff --git a/tests/ExtractFilterRawData/vspace.in b/tests/ExtractFilterRaw/vspace.in similarity index 100% rename from tests/ExtractFilterRawData/vspace.in rename to tests/ExtractFilterRaw/vspace.in diff --git a/tests/ExtractFilterRawData/test_bpextractfilterraw.py b/tests/ExtractFilterRawData/test_bpextractfilterraw.py deleted file mode 100644 index aeb9330..0000000 --- a/tests/ExtractFilterRawData/test_bpextractfilterraw.py +++ /dev/null @@ -1,52 +0,0 @@ -import subprocess -import numpy as np -import os -import pathlib -import warnings -import h5py -import multiprocessing as mp -import sys -import bigplanet as bp - - -def test_bpextract(): - # gets current path - path = pathlib.Path(__file__).parents[0].absolute() - sys.path.insert(1, str(path.parents[0])) - - # gets the number of cores on the machine - cores = mp.cpu_count() - if cores == 1: - warnings.warn("There is only 1 core on the machine", stacklevel=3) - else: - # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / "Test.bpf").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) - - file = bp.BPLFile(path / "Test.bpf") - - earth_Instellation_final = bp.ExtractColumn( - file, 'earth:Instellation:final') - sun_Luminosity_option = bp.ExtractColumn( - file, 'sun:dLuminosity:option') - earth_Mass_option = bp.ExtractColumn(file, 'earth:dMass:option') - vpl_stoptime_option = bp.ExtractColumn(file, 'vpl:dStopTime:option') - earth_tman_forward = bp.ExtractColumn(file, 'earth:TMan:forward') - - assert np.isclose(earth_Instellation_final[1], 341.90883) - assert np.isclose(sun_Luminosity_option[0], 3.846e26) - assert np.isclose(earth_Mass_option[1], -1.5) - assert np.isclose(vpl_stoptime_option[0], 4.5e9) - assert np.isclose(earth_tman_forward[0][0], 3000.0) - - -if __name__ == "__main__": - test_bpextract() diff --git a/tests/MD5CheckSum/test_MD5CheckSum.py b/tests/MD5CheckSum/test_MD5CheckSum.py new file mode 100644 index 0000000..ba4a9bf --- /dev/null +++ b/tests/MD5CheckSum/test_MD5CheckSum.py @@ -0,0 +1,60 @@ +import hashlib +import multiprocessing as mp +import os +import pathlib +import subprocess +import sys +import warnings +import shutil + +def test_MD5CheckSum(): + # gets current path + path = pathlib.Path(__file__).parents[0].absolute() + sys.path.insert(1, str(path.parents[0])) + + # gets the number of cores on the machine + cores = mp.cpu_count() + if cores == 1: + warnings.warn("There is only 1 core on the machine", stacklevel=3) + else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / ".BP_Extract_BPL").exists(): + os.remove(path / ".BP_Extract_BPL") + if (path / "BP_Extract.md5").exists(): + os.remove(path / "BP_Extract.md5") + + # Run vspace + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multi-planet + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run bigplanet + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + + bpa = path / "BP_Extract.bpa" + + md5file = path / "BP_Extract.md5" + with open(md5file, "r") as md5: + array = md5.read().splitlines() + md5_old = array[0] + with open(bpa, "rb") as f: + file_hash = hashlib.md5() + for chunk in iter(lambda: f.read(32768), b""): + file_hash.update(chunk) + new_md5 = file_hash.hexdigest() + assert md5_old == new_md5 + + +if __name__ == "__main__": + test_MD5CheckSum() diff --git a/tests/MD5CheckSum/test_bpmd5checksum.py b/tests/MD5CheckSum/test_bpmd5checksum.py deleted file mode 100644 index ea1c5ac..0000000 --- a/tests/MD5CheckSum/test_bpmd5checksum.py +++ /dev/null @@ -1,49 +0,0 @@ -import subprocess -import numpy as np -import os -import pathlib -import warnings -import h5py -import multiprocessing as mp -import sys -import bigplanet as bp -import hashlib - - -def test_bpextract(): - # gets current path - path = pathlib.Path(__file__).parents[0].absolute() - sys.path.insert(1, str(path.parents[0])) - - # gets the number of cores on the machine - cores = mp.cpu_count() - if cores == 1: - warnings.warn("There is only 1 core on the machine", stacklevel=3) - else: - # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / ".BP_Extract_BPL").exists(): - subprocess.check_output(["bigplanet", "bpl.in", '-a'], cwd=path) - - bpa = path / "BP_Extract.bpa" - - md5file = path / "BP_Extract.md5" - with open(md5file, "r") as md5: - md5_old = md5.readline() - with open(bpa, "rb") as f: - file_hash = hashlib.md5() - for chunk in iter(lambda: f.read(32768), b''): - file_hash.update(chunk) - new_md5 = file_hash.hexdigest() - assert md5_old == new_md5 - - -if __name__ == "__main__": - test_bpextract() diff --git a/tests/SingleSim/test_singlesim.py b/tests/SingleSim/test_singlesim.py index 9e66c27..91abfba 100644 --- a/tests/SingleSim/test_singlesim.py +++ b/tests/SingleSim/test_singlesim.py @@ -1,15 +1,14 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib -import warnings -import h5py -import multiprocessing as mp +import subprocess import sys +import warnings +import shutil +import numpy as np import bigplanet as bp - -def test_singlesim(): +def test_SingleSim(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -19,26 +18,39 @@ def test_singlesim(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / ".BP_Extract_BPL").exists(): + os.remove(path / ".BP_Extract_BPL") + if (path / "BP_Extract.md5").exists(): + os.remove(path / "BP_Extract.md5") + # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / "Test.bpf").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) file = bp.BPLFile(path / "Test.bpf") - earth_Tman_forward = bp.ExtractColumn(file, 'earth:TMan:forward') - earth_Tcore_inital = bp.ExtractColumn(file, 'earth:TCore:initial') + earth_Tman_forward = bp.ExtractColumn(file, "earth:TMan:forward") + earth_Tcore_inital = bp.ExtractColumn(file, "earth:TCore:initial") assert np.isclose(earth_Tman_forward[0][-1], 2257.850930) assert np.isclose(earth_Tcore_inital[0], 6000.00000) if __name__ == "__main__": - test_singlesim() + test_SingleSim() diff --git a/tests/Stats/bpl.in b/tests/Stats/bpl.in index c9498e2..e55f256 100644 --- a/tests/Stats/bpl.in +++ b/tests/Stats/bpl.in @@ -3,5 +3,3 @@ sArchiveFile BP_Stats.bpa saBodyFiles sun.in earth.in sPrimaryFile vpl.in - - diff --git a/tests/Stats/test_Stats.py b/tests/Stats/test_Stats.py new file mode 100644 index 0000000..91588fb --- /dev/null +++ b/tests/Stats/test_Stats.py @@ -0,0 +1,73 @@ +import multiprocessing as mp +import os +import pathlib +import subprocess +import sys +import warnings +import shutil +import numpy as np +import bigplanet as bp + +def test_Stats(): + # gets current path + path = pathlib.Path(__file__).parents[0].absolute() + sys.path.insert(1, str(path.parents[0])) + + # up = os.path.normpath("../") + # if (up / "*.md5").exists: + # os.remove(path / "*.md5") + + + # gets the number of cores on the machine + cores = mp.cpu_count() + if cores == 1: + warnings.warn("There is only 1 core on the machine", stacklevel=3) + else: + # If present, remove files from previous run + if (path / "BP_Stats").exists(): + shutil.rmtree(path / "BP_Stats") + if (path / ".BP_Stats").exists(): + os.remove(path / ".BP_Stats") + if (path / ".BP_Stats_BPL").exists(): + os.remove(path / ".BP_Stats_BPL") + if (path / "BP_Stats.bpa").exists(): + os.remove(path / "BP_Stats.bpa") + if (path / "../BP_Stats.md5").exists(): + os.remove(path / "../BP_Stats.md5") + if (path / "BP_Stats.md5").exists(): + os.remove(path / "BP_Stats.md5") + + # Run vspace + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multi-planet + print("Running MultiPlanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run bigplanet + print("Running BigPlanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + + file = bp.BPLFile(path / "BP_Stats.bpa") + + earth_TMan_min = bp.ExtractColumn(file, "earth:TMan:min") + earth_235UNumMan_max = bp.ExtractColumn(file, "earth:235UNumMan:max") + earth_TCMB_mean = bp.ExtractColumn(file, "earth:TCMB:mean") + earth_FMeltUMan_geomean = bp.ExtractColumn( + file, "earth:FMeltUMan:geomean" + ) + earth_BLUMan_stddev = bp.ExtractColumn(file, "earth:BLUMan:stddev") + + assert np.isclose(earth_TMan_min[0], 2257.85093) + assert np.isclose(earth_235UNumMan_max[0], 2.700598e28) + assert np.isclose(earth_TCMB_mean[0], 4359.67230935255) + assert np.isclose(earth_FMeltUMan_geomean[0], 0.20819565439935903) + assert np.isclose(earth_BLUMan_stddev[0], 18.285373298439122) + + +if __name__ == "__main__": + test_Stats() diff --git a/tests/Stats/test_bpstats.py b/tests/Stats/test_bpstats.py deleted file mode 100644 index 5a3a0ac..0000000 --- a/tests/Stats/test_bpstats.py +++ /dev/null @@ -1,51 +0,0 @@ -import subprocess -import numpy as np -import os -import warnings -import h5py -import multiprocessing as mp -import sys -import pathlib -import bigplanet as bp - - -def test_bpstats(): - # gets current path - path = pathlib.Path(__file__).parents[0].absolute() - sys.path.insert(1, str(path.parents[0])) - - # gets the number of cores on the machine - cores = mp.cpu_count() - if cores == 1: - warnings.warn("There is only 1 core on the machine", stacklevel=3) - else: - # Run vspace - if not (path / "BP_Stats").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_Stats").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / "BP_Stats.bpa").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) - - file = bp.BPLFile(path / "BP_Stats.bpa") - - earth_TMan_min = bp.ExtractColumn(file, 'earth:TMan:min') - earth_235UNumMan_max = bp.ExtractColumn(file, 'earth:235UNumMan:max') - earth_TCMB_mean = bp.ExtractColumn(file, 'earth:TCMB:mean') - earth_FMeltUMan_geomean = bp.ExtractColumn( - file, 'earth:FMeltUMan:geomean') - earth_BLUMan_stddev = bp.ExtractColumn(file, 'earth:BLUMan:stddev') - - assert np.isclose(earth_TMan_min[0], 2257.85093) - assert np.isclose(earth_235UNumMan_max[0], 2.700598e+28) - assert np.isclose(earth_TCMB_mean[0], 4359.67230935255) - assert np.isclose(earth_FMeltUMan_geomean[0], 0.20819565439935903) - assert np.isclose(earth_BLUMan_stddev[0], 18.285373298439122) - - -if __name__ == "__main__": - test_bpstats() diff --git a/tests/ExtractFilterRawData/__init__.py b/tests/Status/__init__.py similarity index 100% rename from tests/ExtractFilterRawData/__init__.py rename to tests/Status/__init__.py diff --git a/tests/Bpstatus/bpl.in b/tests/Status/bpl.in similarity index 100% rename from tests/Bpstatus/bpl.in rename to tests/Status/bpl.in diff --git a/tests/ExtractFilterRawData/earth.in b/tests/Status/earth.in similarity index 100% rename from tests/ExtractFilterRawData/earth.in rename to tests/Status/earth.in diff --git a/tests/ExtractFilterRawData/sun.in b/tests/Status/sun.in similarity index 100% rename from tests/ExtractFilterRawData/sun.in rename to tests/Status/sun.in diff --git a/tests/Bpstatus/test_bpstatus.py b/tests/Status/test_Status.py similarity index 96% rename from tests/Bpstatus/test_bpstatus.py rename to tests/Status/test_Status.py index ebd8df3..4547291 100644 --- a/tests/Bpstatus/test_bpstatus.py +++ b/tests/Status/test_Status.py @@ -1,10 +1,11 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib -import multiprocessing as mp -import warnings +import subprocess import sys +import warnings + +import numpy as np def test_bpstatus(): @@ -30,7 +31,7 @@ def test_bpstatus(): subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) subprocess.check_output(["bpstatus", "vspace.in"], cwd=path) - file = (path / "BP_Status.bpa") + file = path / "BP_Status.bpa" # checks if the bpl files exist assert os.path.isfile(file) == True diff --git a/tests/ExtractFilterRawData/vpl.in b/tests/Status/vpl.in similarity index 100% rename from tests/ExtractFilterRawData/vpl.in rename to tests/Status/vpl.in diff --git a/tests/Bpstatus/vspace.in b/tests/Status/vspace.in similarity index 100% rename from tests/Bpstatus/vspace.in rename to tests/Status/vspace.in diff --git a/tests/UlyssesAggregated/bpl.in b/tests/UlyssesAggregated/bpl.in index f75a68d..e4646d4 100644 --- a/tests/UlyssesAggregated/bpl.in +++ b/tests/UlyssesAggregated/bpl.in @@ -7,4 +7,4 @@ bUlysses True saBodyFiles earth.in sun.in sPrimaryFile vpl.in -saKeyInclude earth:Instellation:final sun:dLuminosity:option earth:dMass:option vpl:dStopTime:option \ No newline at end of file +saKeyInclude earth:Instellation:final sun:dLuminosity:option earth:dMass:option vpl:dStopTime:option diff --git a/tests/UlyssesAggregated/test_UlyssesAggregated.py b/tests/UlyssesAggregated/test_UlyssesAggregated.py index 727bb85..0689416 100644 --- a/tests/UlyssesAggregated/test_UlyssesAggregated.py +++ b/tests/UlyssesAggregated/test_UlyssesAggregated.py @@ -1,15 +1,16 @@ -import subprocess -import numpy as np -import os -import pathlib -import warnings import csv import multiprocessing as mp +import os +import pathlib +import subprocess import sys +import warnings +import shutil +import numpy as np import bigplanet.bp_extract as bp -def test_ulyssesaggregated(): +def test_UlyssesAggregated(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -19,24 +20,36 @@ def test_ulyssesaggregated(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / "User.csv").exists(): + os.remove(path / "User.csv") + # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / "User.csv").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) file = path / "User.csv" data = bp.CSVToDict(file, 1) assert np.isclose( - float(data['earth:Instellation:final'][0]), 1367.635318) + float(data["earth:Instellation:final"][0]), 1367.635318 + ) if __name__ == "__main__": - test_ulyssesaggregated() + test_UlyssesAggregated() diff --git a/tests/UlyssesForward/test_UlyssesForward.py b/tests/UlyssesForward/test_UlyssesForward.py new file mode 100644 index 0000000..ba8498b --- /dev/null +++ b/tests/UlyssesForward/test_UlyssesForward.py @@ -0,0 +1,54 @@ +import multiprocessing as mp +import os +import pathlib +import subprocess +import sys +import warnings +import shutil +import numpy as np +import bigplanet as bp + +def test_UlyssesForward(): + # gets current path + path = pathlib.Path(__file__).parents[0].absolute() + sys.path.insert(1, str(path.parents[0])) + + # gets the number of cores on the machine + cores = mp.cpu_count() + if cores == 1: + warnings.warn("There is only 1 core on the machine", stacklevel=3) + else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / "User.csv").exists(): + os.remove(path / "User.csv") + + # Run vspace + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multi-planet + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run bigplanet + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + + file = path / "User.csv" + + data = bp.CSVToDict(file, 1) + + value = float(data["earth:TCore:forward"][451]) + + assert np.isclose(value, 4999.131849) + + +if __name__ == "__main__": + test_UlyssesForward() diff --git a/tests/UlyssesForward/test_bpulyssessforward.py b/tests/UlyssesForward/test_bpulyssessforward.py deleted file mode 100644 index 0e1ee73..0000000 --- a/tests/UlyssesForward/test_bpulyssessforward.py +++ /dev/null @@ -1,45 +0,0 @@ -import subprocess -import numpy as np -import os -import pathlib -import warnings -import h5py -import multiprocessing as mp -import sys -import bigplanet as bp - - -def test_ulyssesforward(): - # gets current path - path = pathlib.Path(__file__).parents[0].absolute() - sys.path.insert(1, str(path.parents[0])) - - # gets the number of cores on the machine - cores = mp.cpu_count() - if cores == 1: - warnings.warn("There is only 1 core on the machine", stacklevel=3) - else: - # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / "User.csv").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) - - file = path / "User.csv" - - data = bp.CSVToDict(file, 1) - - value = float(data['earth:TCore:forward'][451]) - - assert np.isclose( - value, 4999.131849) - - -if __name__ == "__main__": - test_ulyssesforward()