From 7f248d894a37e2d946ce9eef7e6e60ebae4fd1ea Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Wed, 29 Dec 2021 15:58:46 -0800 Subject: [PATCH 01/30] updated codecov version --- .github/workflows/tests.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d67df4..16d2427 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,16 +8,16 @@ on: jobs: tests: - name: "Run tests on py${{ matrix.python-version }}" + name: 'Run tests on py${{ matrix.python-version }}' runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - - python-version: "3.6" - - python-version: "3.7" - - python-version: "3.8" - - python-version: "3.9" + - python-version: '3.6' + - python-version: '3.7' + - python-version: '3.8' + - python-version: '3.9' steps: - uses: actions/checkout@v2 @@ -41,7 +41,6 @@ jobs: run: | python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet - - name: Install bigplanet id: tools if: steps.install.outcome == 'success' @@ -70,7 +69,7 @@ jobs: comment_mode: update last - name: CodeCov - uses: codecov/codecov-action@v1.5.0 + uses: codecov/codecov-action@v2.1.0 with: files: ./coverage.xml #shell: bash -l {0} From de520d8888cffa2ad02869d6a057fc6d1fc02a82 Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Tue, 25 Jan 2022 15:18:30 -0800 Subject: [PATCH 02/30] updated bigplanet --- .github/workflows/tests.yml | 1 + .gitignore | 1 + bigplanet/bigplanet.py | 2 +- bigplanet/bp_process.py | 3 +-- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 16d2427..e9e85ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,6 +18,7 @@ jobs: - python-version: '3.7' - python-version: '3.8' - python-version: '3.9' + - python-version: '3.10' steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 1e476b1..52f2548 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,4 @@ junit/test-results.xml coverage.xml tests/UlyssesAggregated/User.csv +htmlcov/ \ No newline at end of file diff --git a/bigplanet/bigplanet.py b/bigplanet/bigplanet.py index 97e566c..0fca2c7 100644 --- a/bigplanet/bigplanet.py +++ b/bigplanet/bigplanet.py @@ -42,7 +42,7 @@ def Main(bpInputFile, cores, quiet, overwrite, verbose, archive, deleterawdata, if reply[:1] == "n": print("Files NOT deleted. Exiting.") exit() - if reply[:1] != "n" or reply[:1] != "y": + else: print("User input was not valid. Exiting.") exit() diff --git a/bigplanet/bp_process.py b/bigplanet/bp_process.py index 6c8c919..7be0fe5 100755 --- a/bigplanet/bp_process.py +++ b/bigplanet/bp_process.py @@ -40,7 +40,7 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): # if the line starts with a '(' that means its a variable we need to grab the units if line.startswith('('): fv_param = line[1:line.find(')')].strip() - units = line[line.find('[')+1:line.find(']')].strip() + units = line[line.rfind('[')+1:line.rfind(']')].strip() if not units: units = 'nd' @@ -463,7 +463,6 @@ def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive= for k, v in data.items(): var = k.split(":")[1] - end = k.split(':')[-1] if "OutputOrder" in var or "GridOutput" in var: v_value = v From 81873c0153a803bbd450b642022cbad72a7c359b Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Wed, 26 Jan 2022 12:49:58 -0800 Subject: [PATCH 03/30] updated env yml --- environment.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/environment.yml b/environment.yml index 37a1c00..99a6e22 100644 --- a/environment.yml +++ b/environment.yml @@ -2,22 +2,22 @@ name: vplanet channels: - defaults dependencies: - - numpy==1.19.2 - - pip==20.3.3 - - scipy==1.5.2 - - setuptools==51.3.3 + - numpy>=1.19.2 + - pip>=20.3.3 + - scipy>=1.5.2 + - setuptools>=51.3.3 - pip: - - matplotlib==3.3.3 - - nose==1.3.7 - - parameterized==0.8.1 - - pytest==6.2.4 - - pytest-dependency==0.5.1 - - pytest-env==0.6.2 - - pytest-cov==2.11.1 - - sphinx>=4.0.1 - - breathe==4.30.0 - - sphinxcontrib-bibtex==2.2.0 - - coverage==5.5 - - argparse==1.4 - - h5py==3.1 - - sphinx_rtd_theme==1.0.0 + - matplotlib>=3.3.3 + - nose>=1.3.7 + - parameterized>=0.8.1 + - pytest>=6.2.4 + - pytest-dependency>=0.8.1 + - pytest-env>=0.6.2 + - pytest-cov>=2.11.1 + - sphinx>=4.0.1 + - breathe>=4.30.0 + - sphinxcontrib-bibtex>=2.2.0 + - coverage>=5.5 + - argparse>=1.4 + - h5py>=3.1 + - sphinx_rtd_theme>=1.0.0 From 4c778a99b028cd3553aefc1b8a9c6057c3f6cd42 Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Wed, 26 Jan 2022 12:55:56 -0800 Subject: [PATCH 04/30] updated test yml --- .github/workflows/tests.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e9e85ca..df900c3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,14 +33,16 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} - - name: Install vplanet-private pre-release + - name: Install vplanet id: install if: steps.setup_python.outcome == 'success' shell: bash -l {0} env: - ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - run: | - python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet + #ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + run: + python -m pip install vplanet + + #python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet - name: Install bigplanet id: tools From 53fea7997e5b56a7fc7bfbb9247b62b122258d2f Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Wed, 26 Jan 2022 12:57:41 -0800 Subject: [PATCH 05/30] typo in test yml --- .github/workflows/tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index df900c3..b1caf21 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,10 +39,9 @@ jobs: shell: bash -l {0} env: #ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - run: - python -m pip install vplanet - #python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet + run: | + python -m pip install vplanet - name: Install bigplanet id: tools From 202122b0aa31589e1cac9a123a121504e694ba49 Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Wed, 26 Jan 2022 13:07:49 -0800 Subject: [PATCH 06/30] updated test yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b1caf21..d36df30 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} - - name: Install vplanet + - name: Install VPLanet id: install if: steps.setup_python.outcome == 'success' shell: bash -l {0} From 9d21a94759cfbea0d49d85f7d7f77bcd6baa547c Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Wed, 26 Jan 2022 13:09:18 -0800 Subject: [PATCH 07/30] commented out env --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d36df30..48cca61 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,9 +37,9 @@ jobs: id: install if: steps.setup_python.outcome == 'success' shell: bash -l {0} - env: - #ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - #python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet + #env: + #ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + #python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet run: | python -m pip install vplanet From 2cf89095aa599a24d1e299c0e789fc7a9fc9492f Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Wed, 26 Jan 2022 13:12:10 -0800 Subject: [PATCH 08/30] fixed env yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 99a6e22..b9ed351 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,7 @@ dependencies: - nose>=1.3.7 - parameterized>=0.8.1 - pytest>=6.2.4 - - pytest-dependency>=0.8.1 + - pytest-dependency>=0.5.1 - pytest-env>=0.6.2 - pytest-cov>=2.11.1 - sphinx>=4.0.1 From aff9cf71a36b5ced5179fbc22576b11083db0f27 Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Wed, 26 Jan 2022 14:44:01 -0800 Subject: [PATCH 09/30] updated pip command --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 48cca61..65d095f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,8 +40,7 @@ jobs: #env: #ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} #python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet - run: | - python -m pip install vplanet + run: python -m pip install vplanet - name: Install bigplanet id: tools From 0a1ea1ebd8f20be64631df04e9310ca15038f2e9 Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Fri, 28 Jan 2022 10:03:44 -0800 Subject: [PATCH 10/30] attempt to fix 3.10 install --- .github/workflows/tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65d095f..f5dc845 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,10 +37,7 @@ jobs: id: install if: steps.setup_python.outcome == 'success' shell: bash -l {0} - #env: - #ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - #python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet - run: python -m pip install vplanet + run: python -m pip install vplanet==2.2.10 - name: Install bigplanet id: tools From fb82a22d5939247d8f4140ec8ffd92c240ffcebc Mon Sep 17 00:00:00 2001 From: caitlyn-wilhelm Date: Mon, 31 Jan 2022 12:35:08 -0800 Subject: [PATCH 11/30] ran pre-commit --- .gitignore | 2 +- .pre-commit-config.yaml | 25 ++ README.md | 2 +- bigplanet/bigplanet.py | 113 ++++-- bigplanet/bigplanet_archive.py | 149 +++++--- bigplanet/bigplanet_filter.py | 149 ++++++-- bigplanet/bp_extract.py | 123 ++++--- bigplanet/bp_get.py | 178 ++++++---- bigplanet/bp_process.py | 324 +++++++++++------- bigplanet/bpstatus.py | 29 +- setup.py | 5 +- tests/Bpstatus/test_bpstatus.py | 11 +- tests/CreateHDF5/test_bpcreatehdf5.py | 11 +- tests/ExtractArchive/test_bpextract.py | 15 +- .../test_bpextractfilterarchive.py | 22 +- .../test_bpextractfilterraw.py | 22 +- tests/MD5CheckSum/test_bpmd5checksum.py | 16 +- tests/SingleSim/test_singlesim.py | 14 +- tests/Stats/bpl.in | 2 - tests/Stats/test_bpstats.py | 25 +- tests/UlyssesAggregated/bpl.in | 2 +- .../test_UlyssesAggregated.py | 15 +- .../UlyssesForward/test_bpulyssessforward.py | 15 +- 23 files changed, 815 insertions(+), 454 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.gitignore b/.gitignore index 52f2548..cbd0cac 100644 --- a/.gitignore +++ b/.gitignore @@ -89,4 +89,4 @@ junit/test-results.xml coverage.xml tests/UlyssesAggregated/User.csv -htmlcov/ \ No newline at end of file +htmlcov/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d7ebec1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +exclude: 'docs/' + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/psf/black + rev: 21.12b0 + hooks: + - id: black + verbose: true + exclude: '__init__.py' + + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + args: ['--profile', 'black'] + verbose: true + exclude: '__init__.py' diff --git a/README.md b/README.md index 25a7777..4c023b9 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - +

diff --git a/bigplanet/bigplanet.py b/bigplanet/bigplanet.py index 0fca2c7..254283e 100644 --- a/bigplanet/bigplanet.py +++ b/bigplanet/bigplanet.py @@ -1,19 +1,30 @@ #!/usr/bin/env python -import os +import argparse import multiprocessing as mp +import os import subprocess as sub -import argparse + import h5py import numpy as np import pandas as pd + from .bigplanet_archive import Archive from .bigplanet_filter import Filter -from .bp_get import ReadFile from .bp_extract import Md5CheckSum +from .bp_get import ReadFile -def Main(bpInputFile, cores, quiet, overwrite, verbose, archive, deleterawdata, ignorecorrupt): +def Main( + bpInputFile, + cores, + quiet, + overwrite, + verbose, + archive, + deleterawdata, + ignorecorrupt, +): # folder,bplArchive,output,bodyFileList,primaryFile,IncludeList,ExcludeList,Ulysses = ReadFile(file,verbose) # # if if IncludeList != [] and ExcludeList != [] and os.path.isfile(bplArchive) == False: @@ -23,14 +34,26 @@ def Main(bpInputFile, cores, quiet, overwrite, verbose, archive, deleterawdata, # print("Creating filtered BPF file...") # MainMethodF(bpInputFile,quiet,verbose) if deleterawdata == True: - #folder_name, bpl_file, outputFile, bodylist, primaryFile, includelist, excludelist, Ulysses, SimName - folder, bplArchive, output, bodyFileList, primaryFile, IncludeList, ExcludeList, Ulysses, simname = ReadFile( - bpInputFile, verbose, archive) + # folder_name, bpl_file, outputFile, bodylist, primaryFile, includelist, excludelist, Ulysses, SimName + ( + folder, + bplArchive, + output, + bodyFileList, + primaryFile, + IncludeList, + ExcludeList, + Ulysses, + simname, + ) = ReadFile(bpInputFile, verbose, archive) if os.path.exists(bplArchive) == True: Md5CheckSum(bplArchive, ignorecorrupt) reply = None - question = ("WARNING: This command will delete the " + - folder + " directory along with any MultiPlanet checkpoint files.") + question = ( + "WARNING: This command will delete the " + + folder + + " directory along with any MultiPlanet checkpoint files." + ) while reply not in ("y", "n"): reply = str(input(question + " (y/n): ")).lower().strip() if reply[:1] == "y": @@ -60,27 +83,63 @@ def Main(bpInputFile, cores, quiet, overwrite, verbose, archive, deleterawdata, def Arguments(): max_cores = mp.cpu_count() parser = argparse.ArgumentParser( - description="Extract data from Vplanet simulations") + description="Extract data from Vplanet simulations" + ) + parser.add_argument( + "bpInputFile", help="Name of the biugplanet input file" + ) + parser.add_argument( + "-c", + "--cores", + type=int, + default=max_cores, + help="Number of processors used", + ) + parser.add_argument( + "-o", + "--overwrite", + action="store_true", + help="overwrite file if it already exists", + ) + parser.add_argument( + "-a", + "--archive", + action="store_true", + help="flag for archive file creation", + ) + parser.add_argument( + "-deleterawdata", + "--deleterawdata", + action="store_true", + help="removes source files after creation of Bigplanet files", + ) parser.add_argument( - "bpInputFile", help="Name of the biugplanet input file") - parser.add_argument("-c", "--cores", type=int, - default=max_cores, help="Number of processors used") - parser.add_argument("-o", "--overwrite", action="store_true", - help="overwrite file if it already exists") - parser.add_argument("-a", "--archive", action="store_true", - help="flag for archive file creation") - parser.add_argument("-deleterawdata", "--deleterawdata", action="store_true", - help="removes source files after creation of Bigplanet files") - parser.add_argument("-ignorecorrupt", "--ignorecorrupt", action="store_true", - help="ignore data corruption for MD5 Checksum") + "-ignorecorrupt", + "--ignorecorrupt", + action="store_true", + help="ignore data corruption for MD5 Checksum", + ) # adds the quiet and verbose as mutually exclusive groups group = parser.add_mutually_exclusive_group() - group.add_argument("-q", "--quiet", action="store_true", - help="no output for bigplanet") - group.add_argument("-v", "--verbose", action="store_true", - help="Prints out excess output for bigplanet") + group.add_argument( + "-q", "--quiet", action="store_true", help="no output for bigplanet" + ) + group.add_argument( + "-v", + "--verbose", + action="store_true", + help="Prints out excess output for bigplanet", + ) args = parser.parse_args() - Main(args.bpInputFile, args.cores, args.quiet, - args.overwrite, args.verbose, args.archive, args.deleterawdata, args.ignorecorrupt) + Main( + args.bpInputFile, + args.cores, + args.quiet, + args.overwrite, + args.verbose, + args.archive, + args.deleterawdata, + args.ignorecorrupt, + ) diff --git a/bigplanet/bigplanet_archive.py b/bigplanet/bigplanet_archive.py index 858c7ba..f9309f6 100755 --- a/bigplanet/bigplanet_archive.py +++ b/bigplanet/bigplanet_archive.py @@ -1,22 +1,33 @@ #!/usr/bin/env python -import os +import argparse import multiprocessing as mp +import os import subprocess as sub -import argparse + import h5py import numpy as np import pandas as pd -from .bp_process import * -from .bp_get import * + from .bp_extract import * +from .bp_get import * +from .bp_process import * def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): # Get the directory and list of from the bpl file - dest_folder, bpl_file, outputFile, bodylist, primaryFile, includelist, excludelist, ulysses, SimName = ReadFile( - bpInputFile, verbose, archive=True) + ( + dest_folder, + bpl_file, + outputFile, + bodylist, + primaryFile, + includelist, + excludelist, + ulysses, + SimName, + ) = ReadFile(bpInputFile, verbose, archive=True) # we need to combine the body and primary files into one list infile_list = [] for i in bodylist: @@ -34,7 +45,7 @@ def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): vplanet_help = GetVplanetHelp() # creates the chepoint file name - checkpoint_file = os.getcwd() + '/' + '.' + dest_folder + '_BPL' + checkpoint_file = os.getcwd() + "/" + "." + dest_folder + "_BPL" # Create the checkpoint file to be used to keep track of the groups if os.path.isfile(checkpoint_file) == False: @@ -43,8 +54,9 @@ def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): # if it does exist, it checks for any 0's (sims that didn't complete) and # changes them to -1 to be re-ran else: - ReCreateCP(checkpoint_file, bpInputFile, - quiet, sim_list, dest_folder, force) + ReCreateCP( + checkpoint_file, bpInputFile, quiet, sim_list, dest_folder, force + ) # now that we have everything we need # we save the name of the Master HDF5 file @@ -56,8 +68,23 @@ def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): # for each core, create a process that adds a group to the hdf5 file and adds that to the Master HDF5 file for i in range(cores): - workers.append(mp.Process(target=par_worker, - args=(checkpoint_file, system_name, body_list, log_file, infile_list, quiet, lock, vplanet_help, master_hdf5_file, verbose))) + workers.append( + mp.Process( + target=par_worker, + args=( + checkpoint_file, + system_name, + body_list, + log_file, + infile_list, + quiet, + lock, + vplanet_help, + master_hdf5_file, + verbose, + ), + ) + ) for w in workers: w.start() for w in workers: @@ -67,46 +94,49 @@ def Archive(bpInputFile, cores, quiet, force, ignorecorrupt, verbose): def CreateCP(checkpoint_file, input_file, sims): - with open(checkpoint_file, 'w') as cp: - cp.write('Vspace File: ' + os.getcwd() + '/' + input_file + '\n') - cp.write('Total Number of Simulations: ' + str(len(sims)) + '\n') + with open(checkpoint_file, "w") as cp: + cp.write("Vspace File: " + os.getcwd() + "/" + input_file + "\n") + cp.write("Total Number of Simulations: " + str(len(sims)) + "\n") for f in range(len(sims)): cp.write(sims[f] + " " + "-1 \n") - cp.write('THE END \n') + cp.write("THE END \n") def ReCreateCP(checkpoint_file, input_file, quiet, sims, folder_name, force): datalist = [] - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r") as f: for newline in f: if newline: datalist.append(newline.strip().split()) for l in datalist: - if l[1] == '0': - l[1] = '-1' + if l[1] == "0": + l[1] = "-1" folder = l[0] - with h5py.File(folder_name + '.bpa', "a") as master: - group_name = "/" + folder.split('/')[-1] + with h5py.File(folder_name + ".bpa", "a") as master: + group_name = "/" + folder.split("/")[-1] if group_name in master: if quiet == False: - print("Deleting", group_name, - "from BPL file...") + print( + "Deleting", + group_name, + "from BPL file...", + ) del master[group_name] - with open(checkpoint_file, 'w') as f: + with open(checkpoint_file, "w") as f: for newline in datalist: - f.writelines(' '.join(newline)+'\n') + f.writelines(" ".join(newline) + "\n") - if all(l[1] == '1' for l in datalist[2:-2]) == True: + if all(l[1] == "1" for l in datalist[2:-2]) == True: if quiet == False: print("All Groups in BPL file exist") if force == True: if quiet == False: print("Deleting BPL file...") - os.remove(folder_name + '.bpa') + os.remove(folder_name + ".bpa") if quiet == False: print("Deleting checkpoint file...") os.remove(checkpoint_file) @@ -116,10 +146,21 @@ def ReCreateCP(checkpoint_file, input_file, quiet, sims, folder_name, force): else: if quiet == False: - print('Continuing from Checkpoint...') - - -def par_worker(checkpoint_file, system_name, body_list, log_file, in_files, quiet, lock, vplanet_help, h5_file, verbose): + print("Continuing from Checkpoint...") + + +def par_worker( + checkpoint_file, + system_name, + body_list, + log_file, + in_files, + quiet, + lock, + vplanet_help, + h5_file, + verbose, +): while True: @@ -127,25 +168,25 @@ def par_worker(checkpoint_file, system_name, body_list, log_file, in_files, quie datalist = [] data = {} - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r") as f: for newline in f: datalist.append(newline.strip().split()) - folder = '' + folder = "" for l in datalist: - if l[1] == '-1': + if l[1] == "-1": folder = l[0] - l[1] = '0' + l[1] = "0" break if not folder: lock.release() return - with open(checkpoint_file, 'w') as f: + with open(checkpoint_file, "w") as f: for newline in datalist: - f.writelines(' '.join(newline)+'\n') + f.writelines(" ".join(newline) + "\n") lock.release() @@ -154,35 +195,49 @@ def par_worker(checkpoint_file, system_name, body_list, log_file, in_files, quie lock.acquire() datalist = [] - with open(checkpoint_file, 'r') as f: + with open(checkpoint_file, "r") as f: for newline in f: datalist.append(newline.strip().split()) - group_name = "/" + folder.split('/')[-1] + group_name = "/" + folder.split("/")[-1] # creates the bpl file and reads to make sure the group name is in the file - with h5py.File(h5_file, 'a') as Master: + with h5py.File(h5_file, "a") as Master: # if not then add it if group_name not in Master: if quiet == False: print("Creating", group_name, "...") - data = GatherData(data, system_name, body_list, - log_file, in_files, vplanet_help, folder, verbose) - DictToBP(data, vplanet_help, Master, - verbose, group_name, archive=True) + data = GatherData( + data, + system_name, + body_list, + log_file, + in_files, + vplanet_help, + folder, + verbose, + ) + DictToBP( + data, + vplanet_help, + Master, + verbose, + group_name, + archive=True, + ) for l in datalist: if l[0] == folder: - l[1] = '1' + l[1] = "1" break else: for l in datalist: if l[0] == folder: - l[1] = '1' + l[1] = "1" break - with open(checkpoint_file, 'w') as f: + with open(checkpoint_file, "w") as f: for newline in datalist: - f.writelines(' '.join(newline)+'\n') + f.writelines(" ".join(newline) + "\n") lock.release() diff --git a/bigplanet/bigplanet_filter.py b/bigplanet/bigplanet_filter.py index b1f9977..fa7725b 100755 --- a/bigplanet/bigplanet_filter.py +++ b/bigplanet/bigplanet_filter.py @@ -1,18 +1,20 @@ #!/usr/bin/env python -import os -import multiprocessing as mp -import subprocess as sub import argparse -import h5py -import numpy as np import csv +import multiprocessing as mp +import os import pathlib +import subprocess as sub import sys + +import h5py +import numpy as np import pandas as pd -from .bp_process import * -from .bp_get import * + from .bp_extract import * +from .bp_get import * +from .bp_process import * def SplitsaKey(saKeylist, verbose): @@ -26,13 +28,23 @@ def SplitsaKey(saKeylist, verbose): # loop over the key list for item in saKeylist: # to figure out what list they belong in, we have to rpartion them and look at the last word - spl = item.rpartition(':') - if spl[-1] == "initial" or spl[-1] == "final" or spl[-1] == 'OutputOption' or spl[-1] == 'GridOutputOption': + spl = item.rpartition(":") + if ( + spl[-1] == "initial" + or spl[-1] == "final" + or spl[-1] == "OutputOption" + or spl[-1] == "GridOutputOption" + ): loglist.append(item) # check if its forward or any of the statsitical functions elif ( - spl[-1] == "forward" or spl[-1] == "mean" or spl[-1] == "mode" or spl[-1] == "max" or - spl[-1] == "min" or spl[-1] == "geomean" or spl[-1] == "stddev" + spl[-1] == "forward" + or spl[-1] == "mean" + or spl[-1] == "mode" + or spl[-1] == "max" + or spl[-1] == "min" + or spl[-1] == "geomean" + or spl[-1] == "stddev" ): forwardlist.append(item) # checks if its a body file @@ -54,8 +66,17 @@ def SplitsaKey(saKeylist, verbose): def Filter(file, quiet, verbose, ignorecorrupt, override): - folder, bplArchive, output, bodyFileList, primaryFile, IncludeList, ExcludeList, Ulysses, SimName = ReadFile( - file, verbose=True, archive=False) + ( + folder, + bplArchive, + output, + bodyFileList, + primaryFile, + IncludeList, + ExcludeList, + Ulysses, + SimName, + ) = ReadFile(file, verbose=True, archive=False) vplHelp = GetVplanetHelp() @@ -66,20 +87,32 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): infile_list.append(primaryFile) if os.path.exists(output) and Ulysses == 0: - print("ERROR:", output, "already exists. Please delete", - output, "and try again") + print( + "ERROR:", + output, + "already exists. Please delete", + output, + "and try again", + ) elif os.path.exists(output) and override == True: print("Overriding output file...") sub.run(["rm", output]) # if the bpl archive file does NOT exist, we have to get the data manually if os.path.isfile(bplArchive) == False: - print("WARNING: BPA File does not exist. Obtaining data from source folder. This make take some time...") + print( + "WARNING: BPA File does not exist. Obtaining data from source folder. This make take some time..." + ) # first we need to see what keys go to what vplanet file (ie body file or log file or forward file) if IncludeList: - loglist, optionList, forwardlist, climatelist, backwardlist = SplitsaKey( - IncludeList, verbose) + ( + loglist, + optionList, + forwardlist, + climatelist, + backwardlist, + ) = SplitsaKey(IncludeList, verbose) # now that we have the list of which keys to look for in which files, we can process the files and grab the data if SimName: simList = GetSims(folder, simname=SimName) @@ -97,7 +130,8 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): print("Processing Log file", log_file) # we need to get the system name to get the name of the logfile data = ProcessLogFile( - log_file, data, sim, verbose, incl=IncludeList) + log_file, data, sim, verbose, incl=IncludeList + ) print(data) if optionList: @@ -106,7 +140,8 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): if verbose: print("Processing input file", k) data = ProcessInputfile( - data, k, sim, vplHelp, verbose, incl=IncludeList) + data, k, sim, vplHelp, verbose, incl=IncludeList + ) if forwardlist: print("Forward file data requested") for body in body_names: @@ -121,17 +156,28 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): if outfile in data: forward_name = data[outfile] else: - forward_name = system_name + '.' + body + '.forward' + forward_name = ( + system_name + "." + body + ".forward" + ) - header = [body + ':' + 'OutputOrder'] + header = [body + ":" + "OutputOrder"] heading = {} print("Obtaining Header for Logfile...") heading = ProcessLogFile( - log_file, heading, sim, verbose, incl=header) + log_file, heading, sim, verbose, incl=header + ) print("Processing Forward File", forward_name) data = ProcessOutputfile( - forward_name, data, body, heading, ':forward', sim, verbose, incl=IncludeList) + forward_name, + data, + body, + heading, + ":forward", + sim, + verbose, + incl=IncludeList, + ) if backwardlist: print("Processing Backwards File") @@ -145,43 +191,72 @@ def Filter(file, quiet, verbose, ignorecorrupt, override): if outfile in data: backward_name = data[outfile] else: - backward_name = system_name + '.' + body + '.backward' + backward_name = ( + system_name + "." + body + ".backward" + ) - header = [body + ':' + 'OutputOrder'] + header = [body + ":" + "OutputOrder"] heading = {} heading = ProcessLogFile( - log_file, heading, sim, verbose, incl=header) + log_file, heading, sim, verbose, incl=header + ) data = ProcessOutputfile( - backward_name, data, body, heading, ':backward', sim, verbose, incl=IncludeList) + backward_name, + data, + body, + heading, + ":backward", + sim, + verbose, + incl=IncludeList, + ) if climatelist: for body in body_names: if any(body in s for s in climatelist) == False: continue else: - header = [body + ':' + 'GridOutputOrder'] - climate_name = system_name + '.' + body + '.Climate' + header = [body + ":" + "GridOutputOrder"] + climate_name = ( + system_name + "." + body + ".Climate" + ) heading = {} heading = ProcessLogFile( - log_file, heading, sim, verbose, incl=header) + log_file, heading, sim, verbose, incl=header + ) data = ProcessOutputfile( - climate_name, data, body, heading, ':climate', sim, verbose, incl=IncludeList) + climate_name, + data, + body, + heading, + ":climate", + sim, + verbose, + incl=IncludeList, + ) if Ulysses == 1: DictToCSV(data, ulysses=True) else: - with h5py.File(output, 'w') as filter: + with h5py.File(output, "w") as filter: # Change this to DictToBP <- this reads from Dict to Bigplanet File - DictToBP(data, vplHelp, filter, verbose, - group_name=None, archive=False) + DictToBP( + data, + vplHelp, + filter, + verbose, + group_name=None, + archive=False, + ) # if the bpl file DOES exist, we just need to open it and extract the data to put it in the filter file else: print("Extracting data from BPA File. Please wait...") archive = BPLFile(bplArchive, ignorecorrupt) if Ulysses == 1: if SimName: - ArchiveToCSV(archive, IncludeList, output, - ulysses=1, group=SimName) + ArchiveToCSV( + archive, IncludeList, output, ulysses=1, group=SimName + ) else: # Change this to ArchiveToCSV <- this reads from archive file and exports a CSV ArchiveToCSV(archive, IncludeList, output, ulysses=1) diff --git a/bigplanet/bp_extract.py b/bigplanet/bp_extract.py index fa970f5..a862faf 100755 --- a/bigplanet/bp_extract.py +++ b/bigplanet/bp_extract.py @@ -1,28 +1,30 @@ #!/usr/bin/env python -import multiprocessing as mp -import h5py -import numpy as np -from scipy import stats -import statistics as st import csv -import pandas as pd import hashlib +import multiprocessing as mp import os +import statistics as st import sys + +import h5py +import numpy as np +import pandas as pd +from scipy import stats + from .bp_get import GetVplanetHelp from .bp_process import DictToBP def BPLFile(hf, ignore_corrupt=False): - file = h5py.File(hf, 'r') + file = h5py.File(hf, "r") key_list = list(file.keys()) if ":" not in key_list[0]: Md5CheckSum(hf, ignore_corrupt) - return h5py.File(hf, 'r') + return h5py.File(hf, "r") def ExtractColumn(hf, k): @@ -66,24 +68,24 @@ def ExtractColumn(hf, k): var = k.split(":")[1] - if var == 'OutputOrder' or var == 'GridOutputOrder': + if var == "OutputOrder" or var == "GridOutputOrder": if archive == True: - dataset = hf[key_list[0] + '/' + k] + dataset = hf[key_list[0] + "/" + k] for d in dataset: for value in d: - data.append(value.decode('UTF-8')) + data.append(value.decode("UTF-8")) else: for v in hf[k]: for item in v: - data.append(item.decode('UTF-8')) + data.append(item.decode("UTF-8")) else: aggreg = k.split(":")[2] - if aggreg == 'forward' or aggreg == "backward" or aggreg == "climate": + if aggreg == "forward" or aggreg == "backward" or aggreg == "climate": if archive == True: for key in key_list: - dataset = hf[key + '/' + k] + dataset = hf[key + "/" + k] for d in dataset: data.append(d) else: @@ -91,46 +93,46 @@ def ExtractColumn(hf, k): for d in dataset: data.append(d) - elif aggreg == 'mean': + elif aggreg == "mean": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((st.mean(i))) - elif aggreg == 'stddev': + elif aggreg == "stddev": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((st.stdev(i))) - elif aggreg == 'min': + elif aggreg == "min": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((min(i))) - elif aggreg == 'max': + elif aggreg == "max": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((max(i))) - elif aggreg == 'mode': + elif aggreg == "mode": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((stats.mode(i))) - elif aggreg == 'geomean': + elif aggreg == "geomean": argument = ForwardData(hf, k) # print(argument) for i in argument: data.append((stats.gmean(i))) - elif aggreg == 'initial' or aggreg == 'final' or aggreg == 'option': + elif aggreg == "initial" or aggreg == "final" or aggreg == "option": if archive == True: for key in key_list: - dataset = hf[key + '/' + k] + dataset = hf[key + "/" + k] for d in dataset: data.append(float(d)) else: @@ -138,7 +140,7 @@ def ExtractColumn(hf, k): data.append(float(d)) else: - print('ERROR: Uknown aggregation option: ', aggreg) + print("ERROR: Uknown aggregation option: ", aggreg) exit() return data @@ -176,20 +178,20 @@ def ExtractUnits(hf, k): key_list = list(hf.keys()) if ":" not in key_list[0]: - dataset = hf[key_list[0] + '/' + k] + dataset = hf[key_list[0] + "/" + k] else: dataset = hf[k] - return dataset.attrs.get('Units') + return dataset.attrs.get("Units") def ForwardData(hf, k): data = [] key_list = list(hf.keys()) - forward = k.rpartition(':')[0] + ':forward' + forward = k.rpartition(":")[0] + ":forward" # if hf is an archive file if ":" not in key_list[0]: for key in key_list: - dataset = hf[key + '/' + forward] + dataset = hf[key + "/" + forward] for d in dataset: data.append(d) # for v in d: @@ -210,13 +212,13 @@ def HFD5Decoder(dataset): if "forward" in dataset.name: for value in d: for num in value: - num = num.astype(float, casting='safe') - #data.append(value.astype(float, casting='safe')) + num = num.astype(float, casting="safe") + # data.append(value.astype(float, casting='safe')) else: - d = d.astype(float, casting='safe') - # and now we reshape it the same shape as the original dataset - #shape = dataset.shape - #data = np.reshape(data, shape) + d = d.astype(float, casting="safe") + # and now we reshape it the same shape as the original dataset + # shape = dataset.shape + # data = np.reshape(data, shape) # print(data) # data.tolist() @@ -251,7 +253,7 @@ def ExtractUniqueValues(hf, k): if archive == True: for key in key_list: - dataset = hf[key + '/' + k] + dataset = hf[key + "/" + k] for d in dataset: data.append(float(d.decode("UTF-8"))) else: @@ -290,7 +292,9 @@ def CreateMatrix(xaxis, yaxis, zarray, orientation=1): ynum = len(yaxis) if xnum * ynum != len(zarray): - print("ERROR: Cannot reshape zarray into shape (", xnum, ",", ynum, ")") + print( + "ERROR: Cannot reshape zarray into shape (", xnum, ",", ynum, ")" + ) exit() zmatrix = np.reshape(zarray, (ynum, xnum)) @@ -354,12 +358,19 @@ def ArchiveToFiltered(inputfile, columns, exportfile): units[i] = ExtractUnits(inputfile, i) with h5py.File(exportfile, "a") as f_dest: - f_dest.create_dataset( - i, data=export[i]) - f_dest[i].attrs['Units'] = units[i] - - -def ArchiveToCSV(inputfile, columns, exportfile, delim=" ", header=False, ulysses=0, group=None): + f_dest.create_dataset(i, data=export[i]) + f_dest[i].attrs["Units"] = units[i] + + +def ArchiveToCSV( + inputfile, + columns, + exportfile, + delim=" ", + header=False, + ulysses=0, + group=None, +): """ Writes an Output file in csv format @@ -402,11 +413,11 @@ def ArchiveToCSV(inputfile, columns, exportfile, delim=" ", header=False, ulysse export.append(i) if ulysses == 1: - delim = ',' - exportfile = 'User.csv' + delim = "," + exportfile = "User.csv" if delim == "": - print('ERROR: Delimiter cannot be empty') + print("ERROR: Delimiter cannot be empty") exit() with open(exportfile, "w", newline="") as f: @@ -421,13 +432,15 @@ def ArchiveToCSV(inputfile, columns, exportfile, delim=" ", header=False, ulysse if header == True: writer.writerow(columns) - export = np.array(export, dtype='object').T.tolist() + export = np.array(export, dtype="object").T.tolist() for name in export: for data in name: writer.writerow([data]) -def DictToCSV(dictData, exportfile="bigplanet.csv", delim=" ", header=False, ulysses=0): +def DictToCSV( + dictData, exportfile="bigplanet.csv", delim=" ", header=False, ulysses=0 +): """ Writes an Output file in csv format @@ -459,9 +472,9 @@ def DictToCSV(dictData, exportfile="bigplanet.csv", delim=" ", header=False, uly """ if ulysses == 1: - delim = ',' + delim = "," # headers.append("") - exportfile = 'User.csv' + exportfile = "User.csv" headers = [] for k in dictData.keys(): @@ -475,10 +488,10 @@ def DictToCSV(dictData, exportfile="bigplanet.csv", delim=" ", header=False, uly dictData[k] = [item for sublist in v for item in sublist] if delim == "": - print('ERROR: Delimiter cannot be empty') + print("ERROR: Delimiter cannot be empty") exit() - df = pd.DataFrame.from_dict(dictData, orient='index').transpose() + df = pd.DataFrame.from_dict(dictData, orient="index").transpose() if header == True or ulysses == 1: df.to_csv(exportfile, index=False, header=True) @@ -526,26 +539,26 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): # while chunk := f.read(32768): # file_hash.update(chunk) # if sys.version_info <= (3, 7): - for chunk in iter(lambda: f.read(32768), b''): + for chunk in iter(lambda: f.read(32768), b""): file_hash.update(chunk) - #print("MD5 Checksum:", file_hash.hexdigest()) + # print("MD5 Checksum:", file_hash.hexdigest()) md5.write(file_hash.hexdigest()) else: with open(md5file, "r") as md5: md5_old = md5.readline() - #print("MD5 from " + md5file + ":", md5_old) + # print("MD5 from " + md5file + ":", md5_old) with open(bpa, "rb") as f: file_hash = hashlib.md5() # if sys.version_info >= (3, 8): # while chunk := f.read(32768): # file_hash.update(chunk) # if sys.version_info <= (3, 7): - for chunk in iter(lambda: f.read(32768), b''): + for chunk in iter(lambda: f.read(32768), b""): file_hash.update(chunk) new_md5 = file_hash.hexdigest() - #print("MD5 from " + name + ".bpa: " + new_md5) + # print("MD5 from " + name + ".bpa: " + new_md5) if md5_old == new_md5: print("MD5 Checksum verified") else: diff --git a/bigplanet/bp_get.py b/bigplanet/bp_get.py index ec2afe7..4b498fe 100755 --- a/bigplanet/bp_get.py +++ b/bigplanet/bp_get.py @@ -7,8 +7,8 @@ def DollarSign(m_bl, m_line, m_num, m_file): - '''Process each line looking for $ at the end, continue until no $ present''' - if m_line[-1] == '$': + """Process each line looking for $ at the end, continue until no $ present""" + if m_line[-1] == "$": m_bl.append(m_line[0:-1]) DollarSign(m_bl, m_file[m_num + 1], (m_num + 1), m_file) else: @@ -25,81 +25,98 @@ def ReadFile(bplSplitFile, verbose=False, archive=False): folder_name = "" SimName = "" - with open(bplSplitFile, 'r') as input: + with open(bplSplitFile, "r") as input: # first thing we check is if both include and exclude are in the file bc that is bad - if 'saKeyInclude' and 'saKeyExclude' in input.read(): - print("ERROR: saKeyInclude and saKeyExclude are mutually exclusive") + if "saKeyInclude" and "saKeyExclude" in input.read(): + print( + "ERROR: saKeyInclude and saKeyExclude are mutually exclusive" + ) exit() - with open(bplSplitFile, 'r') as input: + with open(bplSplitFile, "r") as input: # now we loop over the file and get the various inputs content = [line.strip().split() for line in input.readlines()] for num, line in enumerate(content): if line: # we get the folder where the raw data is stored and have the default output file name set - if line[0] == 'sDestFolder': + if line[0] == "sDestFolder": folder_name = line[1] - outputFile = folder_name.split('/')[-1] + "_filtered.bpf" - bpl_file = folder_name.split('/')[-1] + ".bpa" - if line[0] == 'sArchiveFile': + outputFile = folder_name.split("/")[-1] + "_filtered.bpf" + bpl_file = folder_name.split("/")[-1] + ".bpa" + if line[0] == "sArchiveFile": bpl_file = line[1] - if line[0] == 'sOutputFile': + if line[0] == "sOutputFile": outputFile = line[1] - if line[0] == 'sPrimaryFile': + if line[0] == "sPrimaryFile": primaryFile = line[1] - if line[0] == 'bUlysses': + if line[0] == "bUlysses": Ulysses = 1 - if line[0] == 'sSimName': + if line[0] == "sSimName": SimName = line[1] if line[0] == "saBodyFiles": DollarSign(bodylist, line[1:], num, content) bodylist = list(chain.from_iterable(bodylist)) - if line[0] == 'saKeyInclude': + if line[0] == "saKeyInclude": DollarSign(includelist, line[1:], num, content) includelist = list(chain.from_iterable(includelist)) - if line[0] == 'saKeyExclude': + if line[0] == "saKeyExclude": DollarSign(excludelist, line[1:], num, content) excludelist = list(chain.from_iterable(excludelist)) - if not bpl_file and not includelist and archive == False or not bpl_file and not excludelist and archive == False: - print("Error: No BPL Archive file or Include/Exclude List detected.") + if ( + not bpl_file + and not includelist + and archive == False + or not bpl_file + and not excludelist + and archive == False + ): + print( + "Error: No BPL Archive file or Include/Exclude List detected." + ) exit() - #print("WARNING: This may take some time...") + # print("WARNING: This may take some time...") if not folder_name: print("ERROR: No sDestFolder found in bpl.in file") exit() if Ulysses == 1: - outputFile = 'User.csv' - print("WARNING: Ulysses is set to True. Changed Output file to User.csv") + outputFile = "User.csv" + print( + "WARNING: Ulysses is set to True. Changed Output file to User.csv" + ) # if the simName is some value we need to check if all the keys are forward if SimName: for i in includelist: - if 'forward' not in i: + if "forward" not in i: print( - "ERROR: SimName must only use forward file data for output") + "ERROR: SimName must only use forward file data for output" + ) exit() for j in excludelist: - if 'forward' not in j: + if "forward" not in j: print( - "ERROR: SimName must only use forward file data for output") + "ERROR: SimName must only use forward file data for output" + ) exit() if Ulysses == True and SimName == "": for i in includelist: - if 'forward' in i: + if "forward" in i: print( - "ERROR: Forward keys in saKeyInclude requires sSimName for Ulysses Output") + "ERROR: Forward keys in saKeyInclude requires sSimName for Ulysses Output" + ) exit() for j in excludelist: - if 'forward' in j: + if "forward" in j: print( - "ERROR: Forward keys in saKeyExclude requires sSimName for Ulysses Output") + "ERROR: Forward keys in saKeyExclude requires sSimName for Ulysses Output" + ) exit() if verbose: @@ -114,40 +131,60 @@ def ReadFile(bplSplitFile, verbose=False, archive=False): print("Body Files:", bodylist) print("Primary File:", primaryFile) - return folder_name, bpl_file, outputFile, bodylist, primaryFile, includelist, excludelist, Ulysses, SimName + return ( + folder_name, + bpl_file, + outputFile, + bodylist, + primaryFile, + includelist, + excludelist, + Ulysses, + SimName, + ) def GetDir(vspace_file): - """ Give it input file and returns name of folder where simulations are located. """ + """Give it input file and returns name of folder where simulations are located.""" infiles = [] # gets the folder name with all the sims - with open(vspace_file, 'r') as vpl: + with open(vspace_file, "r") as vpl: content = [line.strip().split() for line in vpl.readlines()] for line in content: if line: - if line[0] == 'sDestFolder': + if line[0] == "sDestFolder": folder_name = line[1] - if line[0] == 'sBodyFile' or line[0] == 'sPrimaryFile': + if line[0] == "sBodyFile" or line[0] == "sPrimaryFile": infiles.append(line[1]) if folder_name is None: - raise IOError("Name of destination folder not provided in file '%s'." - "Use syntax 'destfolder '" % vspace_file) + raise IOError( + "Name of destination folder not provided in file '%s'." + "Use syntax 'destfolder '" % vspace_file + ) if os.path.isdir(folder_name) == False: - print("ERROR: Folder", folder_name, - "does not exist in the current directory.") + print( + "ERROR: Folder", + folder_name, + "does not exist in the current directory.", + ) exit() return folder_name, infiles def GetSims(folder_name, simname=""): - """ Pass it folder name where simulations are and returns list of simulation folders. """ + """Pass it folder name where simulations are and returns list of simulation folders.""" # gets the list of sims - sims = sorted([f.path for f in os.scandir( - os.path.abspath(folder_name)) if f.is_dir()]) + sims = sorted( + [ + f.path + for f in os.scandir(os.path.abspath(folder_name)) + if f.is_dir() + ] + ) if simname: sims = [x for x in sims if simname in x] @@ -165,19 +202,19 @@ def GetSNames(bodyfiles, sims): full_path = os.path.join(sims[0], file) # if the infile is the primary file, then get the system name if bodyfiles[-1] == file: - with open(full_path, 'r') as vpl: + with open(full_path, "r") as vpl: content = [line.strip().split() for line in vpl.readlines()] for line in content: if line: - if line[0] == 'sSystemName': + if line[0] == "sSystemName": system_name = line[1] # otherwise get the sName from the body files else: - with open(full_path, 'r') as infile: + with open(full_path, "r") as infile: content = [line.strip().split() for line in infile.readlines()] for line in content: if line: - if line[0] == 'sName': + if line[0] == "sName": body_names.append(line[1]) return system_name, body_names @@ -186,18 +223,33 @@ def GetSNames(bodyfiles, sims): def GetVplanetHelp(): command = "vplanet -H | egrep -v '^$|^\+' | cut -f 2,4 -d '|' | egrep '^ \*\*|^ Cust|^ Type|^ Dim|^ Defa|^Output Parameters'" py_ver = sys.version.split()[0] - if '3.6' in py_ver: - proc = sub.run(command, shell=True, universal_newlines=True, - stdout=sub.PIPE, stderr=sub.PIPE) + if "3.6" in py_ver: + proc = sub.run( + command, + shell=True, + universal_newlines=True, + stdout=sub.PIPE, + stderr=sub.PIPE, + ) else: - proc = sub.run(command, shell=True, text=True, - stdout=sub.PIPE, stderr=sub.PIPE) + proc = sub.run( + command, shell=True, text=True, stdout=sub.PIPE, stderr=sub.PIPE + ) output = proc.stdout.splitlines() vplanet_dict = {} for count, line in enumerate(output): - if ((line.startswith(' **b') or line.startswith(' **d') or line.startswith(' **i') or line.startswith(' **s')) or line.startswith(' **sa') and len(line.split()) == 1): + if ( + ( + line.startswith(" **b") + or line.startswith(" **d") + or line.startswith(" **i") + or line.startswith(" **s") + ) + or line.startswith(" **sa") + and len(line.split()) == 1 + ): option = line.strip("** ") # print("Option:",option) @@ -205,27 +257,27 @@ def GetVplanetHelp(): num = count + 1 while num != count: if "Type" in output[num]: - tp = output[num].rpartition('|')[-1].strip() - vplanet_dict[option]['Type'] = tp + tp = output[num].rpartition("|")[-1].strip() + vplanet_dict[option]["Type"] = tp # print("Type:",tp) num += 1 elif "Custom unit" in output[num]: - custom_unit = output[num].rpartition('|')[-1].strip() - vplanet_dict[option]['Custom Unit'] = custom_unit - #print("Custom Unit:",custom_unit) + custom_unit = output[num].rpartition("|")[-1].strip() + vplanet_dict[option]["Custom Unit"] = custom_unit + # print("Custom Unit:",custom_unit) num += 1 elif "Dimension(s)" in output[num]: - dim = output[num].rpartition('|')[-1].strip() - vplanet_dict[option]['Dimension'] = dim + dim = output[num].rpartition("|")[-1].strip() + vplanet_dict[option]["Dimension"] = dim # print("Dimension:",dim) num += 1 elif "Default value" in output[num]: - default = output[num].rpartition('|')[-1].strip() - vplanet_dict[option]['Default Value'] = default - #print("Default Value",default) + default = output[num].rpartition("|")[-1].strip() + vplanet_dict[option]["Default Value"] = default + # print("Default Value",default) # print() num += 1 @@ -247,11 +299,11 @@ def GetLogName(in_files, sims, system_name): for file in in_files: full_path = os.path.join(sims[0], file) - with open(full_path, 'r') as vpl: + with open(full_path, "r") as vpl: content = [line.strip().split() for line in vpl.readlines()] for line in content: if line: - if line[0] == 'sLogfile': + if line[0] == "sLogfile": prefix = line[1] logfile = prefix + ".log" diff --git a/bigplanet/bp_process.py b/bigplanet/bp_process.py index 7be0fe5..dca6a87 100755 --- a/bigplanet/bp_process.py +++ b/bigplanet/bp_process.py @@ -1,18 +1,19 @@ #!/usr/bin/env python import os -import pandas as pd -import numpy as np + import h5py +import numpy as np +import pandas as pd def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): - prop = '' - body = 'system' + prop = "" + body = "system" path = os.path.join(folder, logfile) if verbose == True: print(path) - with open(path, 'r+', errors='ignore') as log: + with open(path, "r+", errors="ignore") as log: content = [line.strip() for line in log.readlines()] for line in content: @@ -21,32 +22,36 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): continue # First we need to get the body names and if its a inital or final value - if line.startswith('-'): - tmp_line = line.replace( - '-', '').replace(':', '').strip().replace(' ', '_') + if line.startswith("-"): + tmp_line = ( + line.replace("-", "") + .replace(":", "") + .strip() + .replace(" ", "_") + ) - if tmp_line.startswith('INITIAL_SYSTEM_PROPERTIES'): - prop = 'initial' + if tmp_line.startswith("INITIAL_SYSTEM_PROPERTIES"): + prop = "initial" - if tmp_line.startswith('FINAL_SYSTEM_PROPERTIES'): - prop = 'final' - body = 'system' + if tmp_line.startswith("FINAL_SYSTEM_PROPERTIES"): + prop = "final" + body = "system" - if tmp_line.startswith('BODY'): - body = tmp_line[tmp_line.find('_')+1:].strip() + if tmp_line.startswith("BODY"): + body = tmp_line[tmp_line.find("_") + 1 :].strip() continue # if the line starts with a '(' that means its a variable we need to grab the units - if line.startswith('('): - fv_param = line[1:line.find(')')].strip() - units = line[line.rfind('[')+1:line.rfind(']')].strip() + if line.startswith("("): + fv_param = line[1 : line.find(")")].strip() + units = line[line.rfind("[") + 1 : line.rfind("]")].strip() if not units: - units = 'nd' + units = "nd" - fv_value = line[line.find(':')+1:].strip() - key_name = body + ':' + fv_param + ':' + prop + fv_value = line[line.find(":") + 1 :].strip() + key_name = body + ":" + fv_param + ":" + prop # make this into a function if incl is not None: @@ -66,25 +71,25 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): data[key_name] = [units, fv_value] # if the name starts with output order then its a list of variables - if line.startswith('Output Order') and len(line[line.find(':'):]) > 1: - parm_key = line[:line.find(':')].replace(' ', '') - params = line[line.find(':') + 1:].strip().split(']') - key_name = body + ':' + parm_key + if line.startswith("Output Order") and len(line[line.find(":") :]) > 1: + parm_key = line[: line.find(":")].replace(" ", "") + params = line[line.find(":") + 1 :].strip().split("]") + key_name = body + ":" + parm_key out_params = [] for i in params: - var = i[:i.find('[')].strip() - units = i[i.find('[') + 1:] + var = i[: i.find("[")].strip() + units = i[i.find("[") + 1 :] if not units: - units = 'nd' + units = "nd" - if var == '': + if var == "": continue out_params.append([var, units]) - key_name_forward = body + ':' + var + ':forward' + key_name_forward = body + ":" + var + ":forward" # make this into a function if incl is not None: @@ -112,25 +117,28 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): data[key_name] = out_params # if the name starts with grid output order then its a list of variables - if line.startswith('Grid Output Order') and len(line[line.find(':'):]) > 1: - parm_key = line[:line.find(':')].replace(' ', '') - params = line[line.find(':') + 1:].strip().split(']') - key_name = body + ':' + parm_key + if ( + line.startswith("Grid Output Order") + and len(line[line.find(":") :]) > 1 + ): + parm_key = line[: line.find(":")].replace(" ", "") + params = line[line.find(":") + 1 :].strip().split("]") + key_name = body + ":" + parm_key out_params = [] for i in params: - var = i[:i.find('[')].strip() - units = i[i.find('[') + 1:] + var = i[: i.find("[")].strip() + units = i[i.find("[") + 1 :] if not units: - units = 'nd' + units = "nd" - if var == '': + if var == "": continue out_params.append([var, units]) - key_name_climate = body + ':' + var + ':climate' + key_name_climate = body + ":" + var + ":climate" # make this into a function if incl is not None: @@ -158,7 +166,9 @@ def ProcessLogFile(logfile, data, folder, verbose, incl=None, excl=None): return data -def ProcessOutputfile(file, data, body, Output, prefix, folder, verbose, incl=None, excl=None): +def ProcessOutputfile( + file, data, body, Output, prefix, folder, verbose, incl=None, excl=None +): path = os.path.join(folder, file) if verbose == True: @@ -169,8 +179,8 @@ def ProcessOutputfile(file, data, body, Output, prefix, folder, verbose, incl=No for k, v in Output.items(): for num in v: header.append(num[0]) - if num[1] == '': - units.append('nd') + if num[1] == "": + units.append("nd") else: units.append(num[1]) @@ -178,7 +188,7 @@ def ProcessOutputfile(file, data, body, Output, prefix, folder, verbose, incl=No sorted = sorted.to_numpy().transpose().tolist() for i, row in enumerate(sorted): - key_name = body + ':' + header[i] + prefix + key_name = body + ":" + header[i] + prefix # if key_name in data: # data[key_name].append(row) @@ -206,8 +216,10 @@ def ProcessOutputfile(file, data, body, Output, prefix, folder, verbose, incl=No return data -def ProcessSeasonalClimatefile(prefix, data, body, name, folder, verbose, incl=None, excl=None): - file_name = prefix + '.' + name + '.0' +def ProcessSeasonalClimatefile( + prefix, data, body, name, folder, verbose, incl=None, excl=None +): + file_name = prefix + "." + name + ".0" path = os.path.join(folder, "SeasonalClimateFiles/", file_name) if verbose == True: @@ -216,19 +228,23 @@ def ProcessSeasonalClimatefile(prefix, data, body, name, folder, verbose, incl=N sorted = pd.read_csv(path, header=None, delim_whitespace=True).to_numpy() sorted = sorted.transpose().tolist() - key_name = body + ':' + name - units = '' - if (name == 'DailyInsol' or name == 'SeasonalFIn' or - name == 'SeasonalFOut' or name == 'SeasonalDivF'): - units = 'W/m^2' - if name == 'PlanckB': - units = 'W/m^2/K' - if name == 'SeasonalIceBalance': - units = 'kg/m^2/s' - if name == 'SeasonalTemp': - units = 'deg C' - if name == 'SeasonalFMerid': - units = 'W' + key_name = body + ":" + name + units = "" + if ( + name == "DailyInsol" + or name == "SeasonalFIn" + or name == "SeasonalFOut" + or name == "SeasonalDivF" + ): + units = "W/m^2" + if name == "PlanckB": + units = "W/m^2/K" + if name == "SeasonalIceBalance": + units = "kg/m^2/s" + if name == "SeasonalTemp": + units = "deg C" + if name == "SeasonalFMerid": + units = "W" # if key_name not in data: # data[key_name]= [units, sorted] @@ -254,10 +270,12 @@ def ProcessSeasonalClimatefile(prefix, data, body, name, folder, verbose, incl=N return data -def ProcessInputfile(data, in_file, folder, vplanet_help, verbose, incl=None, excl=None): +def ProcessInputfile( + data, in_file, folder, vplanet_help, verbose, incl=None, excl=None +): # set the body name equal to the infile name - body = in_file.partition('.')[0] + body = in_file.partition(".")[0] path = os.path.join(folder, in_file) if verbose == True: print(path) @@ -269,19 +287,19 @@ def ProcessInputfile(data, in_file, folder, vplanet_help, verbose, incl=None, ex # for every line in the array check if the line is blank # or if the line starts with a # next = False - t_line = '' + t_line = "" for num, line in enumerate(content): - if len(line) == 0 or line.startswith('#'): + if len(line) == 0 or line.startswith("#"): continue # if theres a comment in the line we don't want that, so partition the # string and use everything before it - if '#' in line: - line = line.partition('#')[0] + if "#" in line: + line = line.partition("#")[0] # if there's a $ we need to get the next line and append it - if '$' in line: + if "$" in line: next = True - line = line.partition('$')[0] + line = line.partition("$")[0] t_line = t_line + line continue @@ -289,20 +307,20 @@ def ProcessInputfile(data, in_file, folder, vplanet_help, verbose, incl=None, ex next = False t_line = t_line + line line = t_line - t_line = '' + t_line = "" line = line.split() key = line[0] value = line[1] - key = key.replace('-', '') - key_name = body + ':' + key + ':option' + key = key.replace("-", "") + key_name = body + ":" + key + ":option" units = ProcessInfileUnits(key, value, folder, path, vplanet_help) - if 'saOutputOrder' in key_name or 'saGridOutput' in key_name: + if "saOutputOrder" in key_name or "saGridOutput" in key_name: for i in value: - if value[0] == '-': + if value[0] == "-": value = value[1:] # make this into a function @@ -327,78 +345,95 @@ def ProcessInputfile(data, in_file, folder, vplanet_help, verbose, incl=None, ex def ProcessInfileUnits(name, value, folder, in_file, vplanet_help): # check if the value is negative and has a negative option - custom_unit = vplanet_help.get(name, {}).get('Custom Units') - if '-' in value and custom_unit != None: + custom_unit = vplanet_help.get(name, {}).get("Custom Units") + if "-" in value and custom_unit != None: unit = custom_unit return unit else: - dim = vplanet_help.get(name, {}).get('Dimension') + dim = vplanet_help.get(name, {}).get("Dimension") # since pressure and energy arent sUnits, we have to replace them with the dims for each - if dim == None or dim == 'nd': - unit = 'nd' + if dim == None or dim == "nd": + unit = "nd" return unit - if 'pressure' in dim: - dim = dim.replace('pressure', '(mass*length^-1*time^-2)') - if 'energy' in dim: - dim = dim.replace('energy', '(mass*length^2*time^-2)') + if "pressure" in dim: + dim = dim.replace("pressure", "(mass*length^-1*time^-2)") + if "energy" in dim: + dim = dim.replace("energy", "(mass*length^2*time^-2)") # check the options file the value was in and see if the Dimension is there - with open(in_file, 'r+') as infile: + with open(in_file, "r+") as infile: infile_lines = infile.readlines() for infile_line in infile_lines: - if 'sUnitLength' in infile_line and 'length' in dim: - dim = dim.replace('length', infile_line.split()[1]) - if 'sUnitAngle' in infile_line and 'angle' in dim: - dim = dim.replace('angle', infile_line.split()[1]) - if 'sUnitTemp' in infile_line and 'temperature' in dim: - dim = dim.replace('temperature', infile_line.split()[1]) - if 'sUnitMass' in infile_line and 'mass' in dim: - dim = dim.replace('mass', infile_line.split()[1]) - if 'sUnitTime' in infile_line and 'time' in dim: - dim = dim.replace('time', infile_line.split()[1]) + if "sUnitLength" in infile_line and "length" in dim: + dim = dim.replace("length", infile_line.split()[1]) + if "sUnitAngle" in infile_line and "angle" in dim: + dim = dim.replace("angle", infile_line.split()[1]) + if "sUnitTemp" in infile_line and "temperature" in dim: + dim = dim.replace("temperature", infile_line.split()[1]) + if "sUnitMass" in infile_line and "mass" in dim: + dim = dim.replace("mass", infile_line.split()[1]) + if "sUnitTime" in infile_line and "time" in dim: + dim = dim.replace("time", infile_line.split()[1]) if infile_line == infile_lines[-1]: # if its not in the options file, it might be in in the vpl.in file - with open(os.path.join(folder, 'vpl.in'), 'r+') as vplfile: + with open(os.path.join(folder, "vpl.in"), "r+") as vplfile: vpl_lines = vplfile.readlines() for vpl_line in vpl_lines: - if 'sUnitLength' in vpl_line and 'length' in dim: - dim = dim.replace('length', vpl_line.split()[1]) - if 'sUnitAngle' in vpl_line and 'angle' in dim: - dim = dim.replace('angle', vpl_line.split()[1]) - if 'sUnitTemp' in vpl_line and 'temperature' in dim: - dim = dim.replace('temperature', vpl_line.split()[1]) - if 'sUnitMass' in vpl_line and 'mass' in dim: - dim = dim.replace('mass', vpl_line.split()[1]) - if 'sUnitTime' in vpl_line and 'time' in dim: - dim = dim.replace('time', vpl_line.split()[1]) + if "sUnitLength" in vpl_line and "length" in dim: + dim = dim.replace("length", vpl_line.split()[1]) + if "sUnitAngle" in vpl_line and "angle" in dim: + dim = dim.replace("angle", vpl_line.split()[1]) + if "sUnitTemp" in vpl_line and "temperature" in dim: + dim = dim.replace("temperature", vpl_line.split()[1]) + if "sUnitMass" in vpl_line and "mass" in dim: + dim = dim.replace("mass", vpl_line.split()[1]) + if "sUnitTime" in vpl_line and "time" in dim: + dim = dim.replace("time", vpl_line.split()[1]) # the only place left is the default of sUnit of the Dimension if vpl_line == vpl_lines[-1]: - if 'length' in dim: + if "length" in dim: dim = dim.replace( - 'length', vplanet_help['sUnitLength']['Default Value']) - if 'angle' in dim: + "length", + vplanet_help["sUnitLength"]["Default Value"], + ) + if "angle" in dim: dim = dim.replace( - 'angle', vplanet_help['sUnitAngle']['Default Value']) - if 'temperature' in dim: + "angle", + vplanet_help["sUnitAngle"]["Default Value"], + ) + if "temperature" in dim: dim = dim.replace( - 'temperature', vplanet_help['sUnitTemp']['Default Value']) - if 'mass' in dim: + "temperature", + vplanet_help["sUnitTemp"]["Default Value"], + ) + if "mass" in dim: dim = dim.replace( - 'mass', vplanet_help['sUnitMass']['Default Value']) - if 'time' in dim: + "mass", vplanet_help["sUnitMass"]["Default Value"] + ) + if "time" in dim: dim = dim.replace( - 'time', vplanet_help['sUnitTime']['Default Value']) + "time", vplanet_help["sUnitTime"]["Default Value"] + ) unit = dim return unit -def GatherData(data, system_name, body_names, logfile, in_files, vplanet_help, folder, verbose): +def GatherData( + data, + system_name, + body_names, + logfile, + in_files, + vplanet_help, + folder, + verbose, +): """ .... """ @@ -417,7 +452,7 @@ def GatherData(data, system_name, body_names, logfile, in_files, vplanet_help, f gridoutputorder = body + ":GridOutputOrder" # if output order from the log file isn't empty process it if outputorder in data: - #OutputOrder = data[outputorder] + # OutputOrder = data[outputorder] OutputOrder = {} OutputOrder[outputorder] = data[outputorder] @@ -426,39 +461,59 @@ def GatherData(data, system_name, body_names, logfile, in_files, vplanet_help, f file_name = data[Outfile] else: # need to figure out if its forward file or backwards file - forwardOption = in_files[-1].partition( - '.')[0] + ":bDoForward:option" - backwardOption = in_files[-1].partition( - '.')[0] + ":bDoBackward:option" + forwardOption = ( + in_files[-1].partition(".")[0] + ":bDoForward:option" + ) + backwardOption = ( + in_files[-1].partition(".")[0] + ":bDoBackward:option" + ) if forwardOption in data: - file_name = system_name + '.' + body + '.forward' + file_name = system_name + "." + body + ".forward" prefix = ":forward" elif backwardOption in data: - file_name = system_name + '.' + body + '.backward' + file_name = system_name + "." + body + ".backward" prefix = ":backward" data = ProcessOutputfile( - file_name, data, body, OutputOrder, prefix, folder, verbose) + file_name, data, body, OutputOrder, prefix, folder, verbose + ) # now process the grid output order (if it exists) if gridoutputorder in data: GridOutputOrder = {} GridOutputOrder[gridoutputorder] = data[gridoutputorder] - climate_name = system_name + '.' + body + '.Climate' + climate_name = system_name + "." + body + ".Climate" data = ProcessOutputfile( - climate_name, data, body, GridOutputOrder, ':climate', folder, verbose) - prefix = system_name + '.' + body - name = ['DailyInsol', 'PlanckB', 'SeasonalDivF', 'SeasonalFIn', - 'SeasonalFMerid', 'SeasonalFOut', 'SeasonalIceBalance', - 'SeasonalTemp'] + climate_name, + data, + body, + GridOutputOrder, + ":climate", + folder, + verbose, + ) + prefix = system_name + "." + body + name = [ + "DailyInsol", + "PlanckB", + "SeasonalDivF", + "SeasonalFIn", + "SeasonalFMerid", + "SeasonalFOut", + "SeasonalIceBalance", + "SeasonalTemp", + ] for i in range(len(name)): data = ProcessSeasonalClimatefile( - prefix, data, body, name[i], folder, verbose) + prefix, data, body, name[i], folder, verbose + ) return data -def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive=True): +def DictToBP( + data, vplanet_help, h5_file, verbose=False, group_name="", archive=True +): for k, v in data.items(): @@ -466,7 +521,7 @@ def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive= if "OutputOrder" in var or "GridOutput" in var: v_value = v - v_attr = '' + v_attr = "" else: v_attr = v[0] @@ -479,13 +534,16 @@ def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive= tp = "float" else: - if vplanet_help.get(var, {}).get('Type') == 'String' or vplanet_help.get(var, {}).get('Type') == 'String-Array': + if ( + vplanet_help.get(var, {}).get("Type") == "String" + or vplanet_help.get(var, {}).get("Type") == "String-Array" + ): tp = "S" else: tp = "float" if archive == True and group_name: - dataset_name = group_name + '/' + k + dataset_name = group_name + "/" + k else: dataset_name = k @@ -499,4 +557,4 @@ def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive= h5_file.create_dataset(dataset_name, data=v_value) - h5_file[dataset_name].attrs['Units'] = v_attr + h5_file[dataset_name].attrs["Units"] = v_attr diff --git a/bigplanet/bpstatus.py b/bigplanet/bpstatus.py index c648270..4129450 100644 --- a/bigplanet/bpstatus.py +++ b/bigplanet/bpstatus.py @@ -1,46 +1,49 @@ +import argparse import os import sys -import argparse def bpstatus(input_file): - with open(input_file, 'r') as vsf: + with open(input_file, "r") as vsf: vspace_all = vsf.readlines() dest_line = vspace_all[1] folder_name = dest_line.strip().split(None, 1)[1] if folder_name is None: raise IOError( - "Name of destination folder not provided in file '%s'. Use syntax 'destfolder '" % input_file) + "Name of destination folder not provided in file '%s'. Use syntax 'destfolder '" + % input_file + ) count_done = 0 count_todo = 0 count_ip = 0 - checkpoint_file = os.getcwd() + '/.' + folder_name + '_BPL' + checkpoint_file = os.getcwd() + "/." + folder_name + "_BPL" if os.path.isfile(checkpoint_file) == False: raise Exception("BigPlanet must be running prior to using bpstatus") else: - with open(checkpoint_file, 'r') as cp: + with open(checkpoint_file, "r") as cp: content = [line.strip().split() for line in cp.readlines()] for number, line in enumerate(content): status = line[1] - if status == '1': + if status == "1": count_done += 1 - elif status == '-1': + elif status == "-1": count_todo += 1 - elif status == '0': + elif status == "0": count_ip += 1 - print('--BigPlanet Status--') - print('Number of Simulations completed: ' + str(count_done)) - print('Number of Simulations in progress: ' + str(count_ip)) - print('Number of Simulations remaining: ' + str(count_todo)) + print("--BigPlanet Status--") + print("Number of Simulations completed: " + str(count_done)) + print("Number of Simulations in progress: " + str(count_ip)) + print("Number of Simulations remaining: " + str(count_todo)) def main(): parser = argparse.ArgumentParser( - description="Check Status of BigPlanet Data Extraction") + description="Check Status of BigPlanet Data Extraction" + ) parser.add_argument("InputFile", help="Name of the vspace input file") args = parser.parse_args() diff --git a/setup.py b/setup.py index 9afd061..86686c8 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from setuptools import setup import os +from setuptools import setup # Setup! setup( @@ -20,8 +20,7 @@ "write_to": os.path.join("bigplanet", "bigplanet_version.py"), "write_to_template": '__version__ = "{version}"\n', }, - install_requires=["numpy", "h5py", "argparse", - "scipy", "pandas"], + install_requires=["numpy", "h5py", "argparse", "scipy", "pandas"], entry_points={ "console_scripts": [ "bigplanet = bigplanet.bigplanet:Arguments", diff --git a/tests/Bpstatus/test_bpstatus.py b/tests/Bpstatus/test_bpstatus.py index ebd8df3..4547291 100644 --- a/tests/Bpstatus/test_bpstatus.py +++ b/tests/Bpstatus/test_bpstatus.py @@ -1,10 +1,11 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib -import multiprocessing as mp -import warnings +import subprocess import sys +import warnings + +import numpy as np def test_bpstatus(): @@ -30,7 +31,7 @@ def test_bpstatus(): subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) subprocess.check_output(["bpstatus", "vspace.in"], cwd=path) - file = (path / "BP_Status.bpa") + file = path / "BP_Status.bpa" # checks if the bpl files exist assert os.path.isfile(file) == True diff --git a/tests/CreateHDF5/test_bpcreatehdf5.py b/tests/CreateHDF5/test_bpcreatehdf5.py index c46b399..a416f20 100644 --- a/tests/CreateHDF5/test_bpcreatehdf5.py +++ b/tests/CreateHDF5/test_bpcreatehdf5.py @@ -1,10 +1,11 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib -import multiprocessing as mp -import warnings +import subprocess import sys +import warnings + +import numpy as np def test_bpcreatehdf5(): @@ -29,7 +30,7 @@ def test_bpcreatehdf5(): if not (path / ".BP_CreateHDF5_BPL").exists(): subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) - file = (path / "BP_CreateHDF5.bpa") + file = path / "BP_CreateHDF5.bpa" # checks if the bpl files exist assert os.path.isfile(file) == True diff --git a/tests/ExtractArchive/test_bpextract.py b/tests/ExtractArchive/test_bpextract.py index 8b04952..8bcbcf8 100644 --- a/tests/ExtractArchive/test_bpextract.py +++ b/tests/ExtractArchive/test_bpextract.py @@ -1,11 +1,13 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib +import subprocess +import sys import warnings + import h5py -import multiprocessing as mp -import sys +import numpy as np + import bigplanet as bp @@ -34,8 +36,9 @@ def test_bpextract(): file = bp.BPLFile(path / "BP_Extract.bpa") earth_Instellation_final = bp.ExtractColumn( - file, 'earth:Instellation:final') - sun_RotPer_initial = bp.ExtractColumn(file, 'sun:RotPer:initial') + file, "earth:Instellation:final" + ) + sun_RotPer_initial = bp.ExtractColumn(file, "sun:RotPer:initial") assert np.isclose(earth_Instellation_final[0], 1367.635318) assert np.isclose(earth_Instellation_final[1], 341.90883) diff --git a/tests/ExtractFilterArchive/test_bpextractfilterarchive.py b/tests/ExtractFilterArchive/test_bpextractfilterarchive.py index 5c027e6..8c30136 100644 --- a/tests/ExtractFilterArchive/test_bpextractfilterarchive.py +++ b/tests/ExtractFilterArchive/test_bpextractfilterarchive.py @@ -1,11 +1,13 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib +import subprocess +import sys import warnings + import h5py -import multiprocessing as mp -import sys +import numpy as np + import bigplanet as bp @@ -38,12 +40,14 @@ def test_bpextract(): file = bp.BPLFile(path / "Test.bpf") earth_Instellation_final = bp.ExtractColumn( - file, 'earth:Instellation:final') + file, "earth:Instellation:final" + ) sun_Luminosity_option = bp.ExtractColumn( - file, 'sun:dLuminosity:option') - earth_Mass_option = bp.ExtractColumn(file, 'earth:dMass:option') - vpl_stoptime_option = bp.ExtractColumn(file, 'vpl:dStopTime:option') - earth_tman_forward = bp.ExtractColumn(file, 'earth:TMan:forward') + file, "sun:dLuminosity:option" + ) + earth_Mass_option = bp.ExtractColumn(file, "earth:dMass:option") + vpl_stoptime_option = bp.ExtractColumn(file, "vpl:dStopTime:option") + earth_tman_forward = bp.ExtractColumn(file, "earth:TMan:forward") assert np.isclose(earth_Instellation_final[1], 341.90883) assert np.isclose(sun_Luminosity_option[0], 3.846e26) diff --git a/tests/ExtractFilterRawData/test_bpextractfilterraw.py b/tests/ExtractFilterRawData/test_bpextractfilterraw.py index aeb9330..839fff2 100644 --- a/tests/ExtractFilterRawData/test_bpextractfilterraw.py +++ b/tests/ExtractFilterRawData/test_bpextractfilterraw.py @@ -1,11 +1,13 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib +import subprocess +import sys import warnings + import h5py -import multiprocessing as mp -import sys +import numpy as np + import bigplanet as bp @@ -34,12 +36,14 @@ def test_bpextract(): file = bp.BPLFile(path / "Test.bpf") earth_Instellation_final = bp.ExtractColumn( - file, 'earth:Instellation:final') + file, "earth:Instellation:final" + ) sun_Luminosity_option = bp.ExtractColumn( - file, 'sun:dLuminosity:option') - earth_Mass_option = bp.ExtractColumn(file, 'earth:dMass:option') - vpl_stoptime_option = bp.ExtractColumn(file, 'vpl:dStopTime:option') - earth_tman_forward = bp.ExtractColumn(file, 'earth:TMan:forward') + file, "sun:dLuminosity:option" + ) + earth_Mass_option = bp.ExtractColumn(file, "earth:dMass:option") + vpl_stoptime_option = bp.ExtractColumn(file, "vpl:dStopTime:option") + earth_tman_forward = bp.ExtractColumn(file, "earth:TMan:forward") assert np.isclose(earth_Instellation_final[1], 341.90883) assert np.isclose(sun_Luminosity_option[0], 3.846e26) diff --git a/tests/MD5CheckSum/test_bpmd5checksum.py b/tests/MD5CheckSum/test_bpmd5checksum.py index ea1c5ac..3a7f810 100644 --- a/tests/MD5CheckSum/test_bpmd5checksum.py +++ b/tests/MD5CheckSum/test_bpmd5checksum.py @@ -1,13 +1,15 @@ -import subprocess -import numpy as np +import hashlib +import multiprocessing as mp import os import pathlib +import subprocess +import sys import warnings + import h5py -import multiprocessing as mp -import sys +import numpy as np + import bigplanet as bp -import hashlib def test_bpextract(): @@ -30,7 +32,7 @@ def test_bpextract(): # Run bigplanet if not (path / ".BP_Extract_BPL").exists(): - subprocess.check_output(["bigplanet", "bpl.in", '-a'], cwd=path) + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) bpa = path / "BP_Extract.bpa" @@ -39,7 +41,7 @@ def test_bpextract(): md5_old = md5.readline() with open(bpa, "rb") as f: file_hash = hashlib.md5() - for chunk in iter(lambda: f.read(32768), b''): + for chunk in iter(lambda: f.read(32768), b""): file_hash.update(chunk) new_md5 = file_hash.hexdigest() assert md5_old == new_md5 diff --git a/tests/SingleSim/test_singlesim.py b/tests/SingleSim/test_singlesim.py index 9e66c27..11dc65a 100644 --- a/tests/SingleSim/test_singlesim.py +++ b/tests/SingleSim/test_singlesim.py @@ -1,11 +1,13 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib +import subprocess +import sys import warnings + import h5py -import multiprocessing as mp -import sys +import numpy as np + import bigplanet as bp @@ -33,8 +35,8 @@ def test_singlesim(): file = bp.BPLFile(path / "Test.bpf") - earth_Tman_forward = bp.ExtractColumn(file, 'earth:TMan:forward') - earth_Tcore_inital = bp.ExtractColumn(file, 'earth:TCore:initial') + earth_Tman_forward = bp.ExtractColumn(file, "earth:TMan:forward") + earth_Tcore_inital = bp.ExtractColumn(file, "earth:TCore:initial") assert np.isclose(earth_Tman_forward[0][-1], 2257.850930) assert np.isclose(earth_Tcore_inital[0], 6000.00000) diff --git a/tests/Stats/bpl.in b/tests/Stats/bpl.in index c9498e2..e55f256 100644 --- a/tests/Stats/bpl.in +++ b/tests/Stats/bpl.in @@ -3,5 +3,3 @@ sArchiveFile BP_Stats.bpa saBodyFiles sun.in earth.in sPrimaryFile vpl.in - - diff --git a/tests/Stats/test_bpstats.py b/tests/Stats/test_bpstats.py index 5a3a0ac..9525c39 100644 --- a/tests/Stats/test_bpstats.py +++ b/tests/Stats/test_bpstats.py @@ -1,11 +1,13 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os +import pathlib +import subprocess +import sys import warnings + import h5py -import multiprocessing as mp -import sys -import pathlib +import numpy as np + import bigplanet as bp @@ -33,15 +35,16 @@ def test_bpstats(): file = bp.BPLFile(path / "BP_Stats.bpa") - earth_TMan_min = bp.ExtractColumn(file, 'earth:TMan:min') - earth_235UNumMan_max = bp.ExtractColumn(file, 'earth:235UNumMan:max') - earth_TCMB_mean = bp.ExtractColumn(file, 'earth:TCMB:mean') + earth_TMan_min = bp.ExtractColumn(file, "earth:TMan:min") + earth_235UNumMan_max = bp.ExtractColumn(file, "earth:235UNumMan:max") + earth_TCMB_mean = bp.ExtractColumn(file, "earth:TCMB:mean") earth_FMeltUMan_geomean = bp.ExtractColumn( - file, 'earth:FMeltUMan:geomean') - earth_BLUMan_stddev = bp.ExtractColumn(file, 'earth:BLUMan:stddev') + file, "earth:FMeltUMan:geomean" + ) + earth_BLUMan_stddev = bp.ExtractColumn(file, "earth:BLUMan:stddev") assert np.isclose(earth_TMan_min[0], 2257.85093) - assert np.isclose(earth_235UNumMan_max[0], 2.700598e+28) + assert np.isclose(earth_235UNumMan_max[0], 2.700598e28) assert np.isclose(earth_TCMB_mean[0], 4359.67230935255) assert np.isclose(earth_FMeltUMan_geomean[0], 0.20819565439935903) assert np.isclose(earth_BLUMan_stddev[0], 18.285373298439122) diff --git a/tests/UlyssesAggregated/bpl.in b/tests/UlyssesAggregated/bpl.in index f75a68d..e4646d4 100644 --- a/tests/UlyssesAggregated/bpl.in +++ b/tests/UlyssesAggregated/bpl.in @@ -7,4 +7,4 @@ bUlysses True saBodyFiles earth.in sun.in sPrimaryFile vpl.in -saKeyInclude earth:Instellation:final sun:dLuminosity:option earth:dMass:option vpl:dStopTime:option \ No newline at end of file +saKeyInclude earth:Instellation:final sun:dLuminosity:option earth:dMass:option vpl:dStopTime:option diff --git a/tests/UlyssesAggregated/test_UlyssesAggregated.py b/tests/UlyssesAggregated/test_UlyssesAggregated.py index 727bb85..30a3645 100644 --- a/tests/UlyssesAggregated/test_UlyssesAggregated.py +++ b/tests/UlyssesAggregated/test_UlyssesAggregated.py @@ -1,11 +1,13 @@ -import subprocess -import numpy as np -import os -import pathlib -import warnings import csv import multiprocessing as mp +import os +import pathlib +import subprocess import sys +import warnings + +import numpy as np + import bigplanet.bp_extract as bp @@ -35,7 +37,8 @@ def test_ulyssesaggregated(): data = bp.CSVToDict(file, 1) assert np.isclose( - float(data['earth:Instellation:final'][0]), 1367.635318) + float(data["earth:Instellation:final"][0]), 1367.635318 + ) if __name__ == "__main__": diff --git a/tests/UlyssesForward/test_bpulyssessforward.py b/tests/UlyssesForward/test_bpulyssessforward.py index 0e1ee73..e4e1ad9 100644 --- a/tests/UlyssesForward/test_bpulyssessforward.py +++ b/tests/UlyssesForward/test_bpulyssessforward.py @@ -1,11 +1,13 @@ -import subprocess -import numpy as np +import multiprocessing as mp import os import pathlib +import subprocess +import sys import warnings + import h5py -import multiprocessing as mp -import sys +import numpy as np + import bigplanet as bp @@ -35,10 +37,9 @@ def test_ulyssesforward(): data = bp.CSVToDict(file, 1) - value = float(data['earth:TCore:forward'][451]) + value = float(data["earth:TCore:forward"][451]) - assert np.isclose( - value, 4999.131849) + assert np.isclose(value, 4999.131849) if __name__ == "__main__": From 80dc6cc801b522e73f3b18dc7ebe199a47c82d57 Mon Sep 17 00:00:00 2001 From: rudyg3 Date: Mon, 25 Apr 2022 14:15:22 -0700 Subject: [PATCH 12/30] Edited create_dataset arguments to handle strings in data --- bigplanet/bp_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigplanet/bp_process.py b/bigplanet/bp_process.py index 6c8c919..6f8129f 100755 --- a/bigplanet/bp_process.py +++ b/bigplanet/bp_process.py @@ -498,6 +498,6 @@ def DictToBP(data, vplanet_help, h5_file, verbose=False, group_name="", archive= print("Value:", v_value) print() - h5_file.create_dataset(dataset_name, data=v_value) + h5_file.create_dataset(dataset_name, data=np.array([v_value], dtype=tp)) h5_file[dataset_name].attrs['Units'] = v_attr From a43b3808f0db590b91f2eb96c41b12270a8328ac Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Mon, 25 Apr 2022 18:16:05 -0700 Subject: [PATCH 13/30] Changed vplanet repo to public in test.yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d67df4..bc10eff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,7 +39,7 @@ jobs: env: ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} run: | - python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet-private.git@main#egg=vplanet + python -m pip install git+https://$ACCESS_TOKEN@github.com/VirtualPlanetaryLaboratory/vplanet.git@main#egg=vplanet - name: Install bigplanet From 51396bf3303cb4b43aecf72b2ca8de0dd942a662 Mon Sep 17 00:00:00 2001 From: RoryBarnes Date: Fri, 26 Aug 2022 15:05:43 -0700 Subject: [PATCH 14/30] Fixed typos in docs. --- docs/Script.rst | 4 ++-- docs/quickstart.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/Script.rst b/docs/Script.rst index 6ce9ef8..679411a 100644 --- a/docs/Script.rst +++ b/docs/Script.rst @@ -37,7 +37,7 @@ where: *Key* is the name of the particular variable you are extracting the units from. -See the `Understanding Keys`_ Section for an indepth look at the types of key options available. +See the `Understanding Keys `_ Section for an indepth look at the types of key options available. @@ -56,7 +56,7 @@ where: *Key* is the name of the particular variable you are extracting the units from. -See the `Understanding Keys`_ Section for an indepth look at the types of key options available. +See the `Understanding Keys `_ Section for an indepth look at the types of key options available. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 9f89699..2fc4c73 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,7 +22,7 @@ This will read in the instructions in the ``bpl.in`` file and the ``-a`` flag te sArchiveFile GDwarf.bpa sOutputFile ice_states.bpf - saBodyFiles earth.in sun.in + saBodyFiles earth.in sPrimaryFile vpl.in saKeyInclude earth:Obliquity:forward earth:Instellation:final earth:IceBeltLand:final $ @@ -49,7 +49,7 @@ extract the appropriate columns from the raw data, but from the archive. Line 3 is the name of the BigPlanet file that will contain the data subset. -Line 5 is the list of body files from which to extract the columns *XXX should sun.in be here?? XXX* +Line 5 is the list of body files from which to extract the columns. Line 6 is the name of primary file (in this case its vpl.in). From 48e415a1a8922f10f4975fea4c3816b1441f6fcc Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Wed, 15 Mar 2023 11:51:07 -0700 Subject: [PATCH 15/30] Update tests.yml --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36e7623..de8ff6e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,9 +2,9 @@ name: tests on: push: - branches: [master, main] + branches: [main] pull_request: - branches: [master, main] + branches: [main] jobs: tests: @@ -29,7 +29,7 @@ jobs: - name: Set up Python id: setup_python - uses: conda-incubator/setup-miniconda@v2 + uses: actions/setup-python@v4 with: activate-environment: vplanet environment-file: environment.yml From 8bede88f2b4feea04df49b0248b148a035f8277a Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Wed, 15 Mar 2023 11:59:11 -0700 Subject: [PATCH 16/30] Update tests.yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index de8ff6e..f2b71a9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,7 +29,7 @@ jobs: - name: Set up Python id: setup_python - uses: actions/setup-python@v4 + uses: conda-incubator/setup-miniconda@v2 with: activate-environment: vplanet environment-file: environment.yml From c69da90be39e3ec55b6874200a172b863e0ed128 Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Wed, 15 Mar 2023 12:05:10 -0700 Subject: [PATCH 17/30] Update tests.yml --- .github/workflows/tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f2b71a9..c3ed29b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,9 +18,6 @@ jobs: - python-version: "3.7" - python-version: "3.8" - python-version: "3.9" - - python-version: "3.10" - - python-version: "3.11" - - python-version: "3.12" steps: - uses: actions/checkout@v2 From be2ebfd00b79354f630afa1cd44776f323aaafec Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Wed, 15 Mar 2023 12:05:26 -0700 Subject: [PATCH 18/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b35f241..15e3c0f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

- + From 78ba1ce78f17a28b8dde853992106525cd5f0065 Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Wed, 15 Mar 2023 12:31:51 -0700 Subject: [PATCH 19/30] Update docs.yml --- .github/workflows/docs.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 478eea8..37ab521 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,16 +2,14 @@ name: docs on: push: - branches: [master, main] - #pull_request: TODO: Enable me - # branches:[master, main] + branches: [main] jobs: tests: name: "Build docs" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python id: setup_python From 40271271552ad354e313f91d4e8d598b48033349 Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Wed, 15 Mar 2023 12:32:55 -0700 Subject: [PATCH 20/30] Update README.md --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 15e3c0f..5623404 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,14 @@

- - + + + + - - + + +

From 9a7efef911e8ec1820e0ed2f9574713f7a9ecafb Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Mon, 20 Mar 2023 13:48:24 -0700 Subject: [PATCH 21/30] Update environment.yml --- environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yml b/environment.yml index 0b15a7d..7cefd13 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,6 @@ dependencies: - scipy>=1.5.2 - setuptools>=51.3.3 - pip: - - matplotlib==3.3.3 - nose==1.3.7 - parameterized==0.8.1 From f2a863cd95f80983ddac2f6bbcae6629b942a2fa Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Mon, 20 Mar 2023 13:55:14 -0700 Subject: [PATCH 22/30] Update environment.yml --- environment.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/environment.yml b/environment.yml index 7cefd13..6b8a316 100644 --- a/environment.yml +++ b/environment.yml @@ -7,17 +7,17 @@ dependencies: - scipy>=1.5.2 - setuptools>=51.3.3 - pip: - - matplotlib==3.3.3 - - nose==1.3.7 - - parameterized==0.8.1 - - pytest==6.2.4 - - pytest-dependency==0.5.1 - - pytest-env==0.6.2 - - pytest-cov==2.11.1 - - sphinx==4.2.0 - - breathe==4.30.0 - - sphinxcontrib-bibtex==2.2.0 - - coverage==5.5 - - argparse==1.4 - - h5py==3.1 - - sphinx_rtd_theme==1.0.0 + - matplotlib>=3.3.3 + - nose>=1.3.7 + - parameterized==0.8.1 + - pytest>=6.2.4 + - pytest-dependency>=0.5.1 + - pytest-env>=0.6.2 + - pytest-cov>=2.11.1 + - sphinx==4.2.0 + - breathe>=4.30.0 + - sphinxcontrib-bibtex>=2.2.0 + - coverage>=5.5 + - argparse>=1.4 + - h5py>=3.1 + - sphinx_rtd_theme==1.0.0 From 7a831a32ff7117f189ae2fe217dee5fb87163b5f Mon Sep 17 00:00:00 2001 From: RoryBarnes Date: Mon, 20 Mar 2023 14:33:59 -0700 Subject: [PATCH 23/30] Fixed issue with md5 checksum. I don't know how this test passed before. --- tests/BP_Extract.md5 | 1 + tests/MD5CheckSum/test_bpmd5checksum.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests/BP_Extract.md5 diff --git a/tests/BP_Extract.md5 b/tests/BP_Extract.md5 new file mode 100644 index 0000000..7fe7b06 --- /dev/null +++ b/tests/BP_Extract.md5 @@ -0,0 +1 @@ +7e71b106f5f8be333fb2d19687e0b2e4 \ No newline at end of file diff --git a/tests/MD5CheckSum/test_bpmd5checksum.py b/tests/MD5CheckSum/test_bpmd5checksum.py index 3a7f810..4f75b92 100644 --- a/tests/MD5CheckSum/test_bpmd5checksum.py +++ b/tests/MD5CheckSum/test_bpmd5checksum.py @@ -38,7 +38,8 @@ def test_bpextract(): md5file = path / "BP_Extract.md5" with open(md5file, "r") as md5: - md5_old = md5.readline() + array = md5.read().splitlines() + md5_old = array[0] with open(bpa, "rb") as f: file_hash = hashlib.md5() for chunk in iter(lambda: f.read(32768), b""): From 6686b82bebc90a3bdaaf87f795dc5d33af3cb51b Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Mon, 20 Mar 2023 20:16:14 -0700 Subject: [PATCH 24/30] Fixing ExtractArchive test. --- bigplanet/bp_extract.py | 9 +++++++-- tests/BP_Extract.md5 | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bigplanet/bp_extract.py b/bigplanet/bp_extract.py index a862faf..0317bfd 100755 --- a/bigplanet/bp_extract.py +++ b/bigplanet/bp_extract.py @@ -529,9 +529,11 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): name = os.path.basename(archivefile.name).split(".")[0] bpa = archivefile + #print("Name: "+name) md5file = name + ".md5" # if it doesn't exist, we need to create it if os.path.isfile(md5file) == False: + print(md5file+" file not found. Generating.") with open(md5file, "w") as md5: with open(bpa, "rb") as f: file_hash = hashlib.md5() @@ -546,8 +548,9 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): md5.write(file_hash.hexdigest()) else: with open(md5file, "r") as md5: + #print("Found "+md5file) md5_old = md5.readline() - # print("MD5 from " + md5file + ":", md5_old) + #print("MD5 from " + md5file + ":", md5_old) with open(bpa, "rb") as f: file_hash = hashlib.md5() # if sys.version_info >= (3, 8): @@ -558,7 +561,7 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): file_hash.update(chunk) new_md5 = file_hash.hexdigest() - # print("MD5 from " + name + ".bpa: " + new_md5) + #print("MD5 from " + name + ".bpa: " + new_md5) if md5_old == new_md5: print("MD5 Checksum verified") else: @@ -566,5 +569,7 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): print("WARNING: MD5 Checksum failed") else: print("ERROR: MD5 Checksum failed") + print("MD5 from " + md5file + ":", md5_old) + print("MD5 from " + name + ".bpa: " + new_md5) print("set flag --ignorecorrupt to still use corrupted data") exit() diff --git a/tests/BP_Extract.md5 b/tests/BP_Extract.md5 index 7fe7b06..369e98a 100644 --- a/tests/BP_Extract.md5 +++ b/tests/BP_Extract.md5 @@ -1 +1 @@ -7e71b106f5f8be333fb2d19687e0b2e4 \ No newline at end of file +bb3e5ea8fe70c0e2a9f7e34798400168 \ No newline at end of file From dd411d7b65e2d344676d5f0a23b0cdd07d1d4ffc Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Mon, 20 Mar 2023 21:53:42 -0700 Subject: [PATCH 25/30] Change test bpextractfilterraw to remove old files before performing test. --- bigplanet/bp_extract.py | 4 +-- .../test_bpextractfilterraw.py | 31 ++++++++++++------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/bigplanet/bp_extract.py b/bigplanet/bp_extract.py index 0317bfd..e41322f 100755 --- a/bigplanet/bp_extract.py +++ b/bigplanet/bp_extract.py @@ -40,8 +40,8 @@ def ExtractColumn(hf, k): k : str the name of the column that is to be extracted Example: - k = 'earth_Obliquity_final' - The syntax of the column names is body_variable_aggregation + k = 'earth:Obliquity:final' + The syntax of the column names is body:variable:aggregation the lists of aggregations (and how to call them) is as follows: forward file data (forward), initial data (initial), diff --git a/tests/ExtractFilterRawData/test_bpextractfilterraw.py b/tests/ExtractFilterRawData/test_bpextractfilterraw.py index 839fff2..cc19865 100644 --- a/tests/ExtractFilterRawData/test_bpextractfilterraw.py +++ b/tests/ExtractFilterRawData/test_bpextractfilterraw.py @@ -4,7 +4,7 @@ import subprocess import sys import warnings - +import shutil import h5py import numpy as np @@ -21,17 +21,23 @@ def test_bpextract(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: - # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / "Test.bpf").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + print("Removing BP_Extract") + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + print("Removing .BP_Extract") + os.remove(path / ".BP_Extract") + if (path / "Test.bpf").exists(): + print("Removing Test.bpf") + os.remove(path / "Test.bpf") + + print("Running vspace") + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running multiplanet") + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running bigplanet") + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) file = bp.BPLFile(path / "Test.bpf") @@ -45,6 +51,7 @@ def test_bpextract(): vpl_stoptime_option = bp.ExtractColumn(file, "vpl:dStopTime:option") earth_tman_forward = bp.ExtractColumn(file, "earth:TMan:forward") + print(earth_Instellation_final[1]) assert np.isclose(earth_Instellation_final[1], 341.90883) assert np.isclose(sun_Luminosity_option[0], 3.846e26) assert np.isclose(earth_Mass_option[1], -1.5) From 056370feeed0fb2066dc654b5c7e5782e5cd2edb Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Fri, 21 Apr 2023 12:07:55 -0700 Subject: [PATCH 26/30] Renamed tests files to match directory names. --- tests/CreateHDF5/{test_bpcreatehdf5.py => test_CreateHDF5.py} | 0 .../ExtractArchive/{test_bpextract.py => test_ExtractArchive.py} | 0 ...est_bpextractfilterarchive.py => test_ExtractFilterArchive.py} | 0 tests/{Bpstatus => ExtractFilterRaw}/__init__.py | 0 tests/{ExtractFilterRawData => ExtractFilterRaw}/bpl.in | 0 tests/{Bpstatus => ExtractFilterRaw}/earth.in | 0 tests/{Bpstatus => ExtractFilterRaw}/sun.in | 0 .../test_ExtractFilterRaw.py} | 0 tests/{Bpstatus => ExtractFilterRaw}/vpl.in | 0 tests/{ExtractFilterRawData => ExtractFilterRaw}/vspace.in | 0 tests/MD5CheckSum/{test_bpmd5checksum.py => test_MD5CheckSum.py} | 0 tests/Stats/{test_bpstats.py => test_Stats.py} | 0 tests/{ExtractFilterRawData => Status}/__init__.py | 0 tests/{Bpstatus => Status}/bpl.in | 0 tests/{ExtractFilterRawData => Status}/earth.in | 0 tests/{ExtractFilterRawData => Status}/sun.in | 0 tests/{Bpstatus/test_bpstatus.py => Status/test_Status.py} | 0 tests/{ExtractFilterRawData => Status}/vpl.in | 0 tests/{Bpstatus => Status}/vspace.in | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename tests/CreateHDF5/{test_bpcreatehdf5.py => test_CreateHDF5.py} (100%) rename tests/ExtractArchive/{test_bpextract.py => test_ExtractArchive.py} (100%) rename tests/ExtractFilterArchive/{test_bpextractfilterarchive.py => test_ExtractFilterArchive.py} (100%) rename tests/{Bpstatus => ExtractFilterRaw}/__init__.py (100%) rename tests/{ExtractFilterRawData => ExtractFilterRaw}/bpl.in (100%) rename tests/{Bpstatus => ExtractFilterRaw}/earth.in (100%) rename tests/{Bpstatus => ExtractFilterRaw}/sun.in (100%) rename tests/{ExtractFilterRawData/test_bpextractfilterraw.py => ExtractFilterRaw/test_ExtractFilterRaw.py} (100%) rename tests/{Bpstatus => ExtractFilterRaw}/vpl.in (100%) rename tests/{ExtractFilterRawData => ExtractFilterRaw}/vspace.in (100%) rename tests/MD5CheckSum/{test_bpmd5checksum.py => test_MD5CheckSum.py} (100%) rename tests/Stats/{test_bpstats.py => test_Stats.py} (100%) rename tests/{ExtractFilterRawData => Status}/__init__.py (100%) rename tests/{Bpstatus => Status}/bpl.in (100%) rename tests/{ExtractFilterRawData => Status}/earth.in (100%) rename tests/{ExtractFilterRawData => Status}/sun.in (100%) rename tests/{Bpstatus/test_bpstatus.py => Status/test_Status.py} (100%) rename tests/{ExtractFilterRawData => Status}/vpl.in (100%) rename tests/{Bpstatus => Status}/vspace.in (100%) diff --git a/tests/CreateHDF5/test_bpcreatehdf5.py b/tests/CreateHDF5/test_CreateHDF5.py similarity index 100% rename from tests/CreateHDF5/test_bpcreatehdf5.py rename to tests/CreateHDF5/test_CreateHDF5.py diff --git a/tests/ExtractArchive/test_bpextract.py b/tests/ExtractArchive/test_ExtractArchive.py similarity index 100% rename from tests/ExtractArchive/test_bpextract.py rename to tests/ExtractArchive/test_ExtractArchive.py diff --git a/tests/ExtractFilterArchive/test_bpextractfilterarchive.py b/tests/ExtractFilterArchive/test_ExtractFilterArchive.py similarity index 100% rename from tests/ExtractFilterArchive/test_bpextractfilterarchive.py rename to tests/ExtractFilterArchive/test_ExtractFilterArchive.py diff --git a/tests/Bpstatus/__init__.py b/tests/ExtractFilterRaw/__init__.py similarity index 100% rename from tests/Bpstatus/__init__.py rename to tests/ExtractFilterRaw/__init__.py diff --git a/tests/ExtractFilterRawData/bpl.in b/tests/ExtractFilterRaw/bpl.in similarity index 100% rename from tests/ExtractFilterRawData/bpl.in rename to tests/ExtractFilterRaw/bpl.in diff --git a/tests/Bpstatus/earth.in b/tests/ExtractFilterRaw/earth.in similarity index 100% rename from tests/Bpstatus/earth.in rename to tests/ExtractFilterRaw/earth.in diff --git a/tests/Bpstatus/sun.in b/tests/ExtractFilterRaw/sun.in similarity index 100% rename from tests/Bpstatus/sun.in rename to tests/ExtractFilterRaw/sun.in diff --git a/tests/ExtractFilterRawData/test_bpextractfilterraw.py b/tests/ExtractFilterRaw/test_ExtractFilterRaw.py similarity index 100% rename from tests/ExtractFilterRawData/test_bpextractfilterraw.py rename to tests/ExtractFilterRaw/test_ExtractFilterRaw.py diff --git a/tests/Bpstatus/vpl.in b/tests/ExtractFilterRaw/vpl.in similarity index 100% rename from tests/Bpstatus/vpl.in rename to tests/ExtractFilterRaw/vpl.in diff --git a/tests/ExtractFilterRawData/vspace.in b/tests/ExtractFilterRaw/vspace.in similarity index 100% rename from tests/ExtractFilterRawData/vspace.in rename to tests/ExtractFilterRaw/vspace.in diff --git a/tests/MD5CheckSum/test_bpmd5checksum.py b/tests/MD5CheckSum/test_MD5CheckSum.py similarity index 100% rename from tests/MD5CheckSum/test_bpmd5checksum.py rename to tests/MD5CheckSum/test_MD5CheckSum.py diff --git a/tests/Stats/test_bpstats.py b/tests/Stats/test_Stats.py similarity index 100% rename from tests/Stats/test_bpstats.py rename to tests/Stats/test_Stats.py diff --git a/tests/ExtractFilterRawData/__init__.py b/tests/Status/__init__.py similarity index 100% rename from tests/ExtractFilterRawData/__init__.py rename to tests/Status/__init__.py diff --git a/tests/Bpstatus/bpl.in b/tests/Status/bpl.in similarity index 100% rename from tests/Bpstatus/bpl.in rename to tests/Status/bpl.in diff --git a/tests/ExtractFilterRawData/earth.in b/tests/Status/earth.in similarity index 100% rename from tests/ExtractFilterRawData/earth.in rename to tests/Status/earth.in diff --git a/tests/ExtractFilterRawData/sun.in b/tests/Status/sun.in similarity index 100% rename from tests/ExtractFilterRawData/sun.in rename to tests/Status/sun.in diff --git a/tests/Bpstatus/test_bpstatus.py b/tests/Status/test_Status.py similarity index 100% rename from tests/Bpstatus/test_bpstatus.py rename to tests/Status/test_Status.py diff --git a/tests/ExtractFilterRawData/vpl.in b/tests/Status/vpl.in similarity index 100% rename from tests/ExtractFilterRawData/vpl.in rename to tests/Status/vpl.in diff --git a/tests/Bpstatus/vspace.in b/tests/Status/vspace.in similarity index 100% rename from tests/Bpstatus/vspace.in rename to tests/Status/vspace.in From 0d9a10290be0b79ae43fdb85c1624a454f08981a Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Fri, 21 Apr 2023 17:36:37 -0700 Subject: [PATCH 27/30] Revamped naming conventions for tests. Tests now delete all old files at start up to prevent accidentally using incorrect files. --- .gitignore | 23 -------- bigplanet/bp_extract.py | 8 +-- tests/CreateHDF5/test_CreateHDF5.py | 32 ++++++++--- tests/ExtractArchive/test_ExtractArchive.py | 35 ++++++++---- .../test_ExtractFilterArchive.py | 40 +++++++++----- .../ExtractFilterRaw/test_ExtractFilterRaw.py | 15 ++++-- tests/MD5CheckSum/test_MD5CheckSum.py | 36 ++++++++----- tests/SingleSim/test_singlesim.py | 34 +++++++----- tests/Stats/test_Stats.py | 36 ++++++++----- .../test_UlyssesAggregated.py | 30 +++++++---- tests/UlyssesForward/test_UlyssesForward.py | 54 +++++++++++++++++++ .../UlyssesForward/test_bpulyssessforward.py | 46 ---------------- 12 files changed, 231 insertions(+), 158 deletions(-) create mode 100644 tests/UlyssesForward/test_UlyssesForward.py delete mode 100644 tests/UlyssesForward/test_bpulyssessforward.py diff --git a/.gitignore b/.gitignore index cbd0cac..ee4243d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,29 +21,6 @@ SeasonalClimateFiles *.csv *.md5 -#files created from running tests -tests/Bpstatus/BP_Status/ -tests/Bpstatus/.* -tests/CreateHDF5/BP_CreateHDF5/ -tests/CreateHDF5/.* -tests/ExtractArchive/BP_Extract/ -tests/ExtractArchive/.* -tests/Stats/BP_Stats/ -tests/Stats/.* -tests/ExtractFilterArchive/BP_Extract/ -tests/ExtractFilterArchive/.* -tests/ExtractFilterRawData/BP_Extract/ -tests/ExtractFilterRawData/.* -tests/MD5CheckSum/BP_Extract/ -tests/MD5CheckSum/.* -tests/UlyssesAggregated/BP_Extract/ -tests/UlyssesAggregated/.* -tests/UlyssesForward/BP_Extract/ -tests/UlyssesForward/.* -tests/SingleSim/BP_Extract/ -tests/SingleSim/.* - - .coverage # Aux files / build info / garbage diff --git a/bigplanet/bp_extract.py b/bigplanet/bp_extract.py index e41322f..ca74887 100755 --- a/bigplanet/bp_extract.py +++ b/bigplanet/bp_extract.py @@ -563,13 +563,13 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): new_md5 = file_hash.hexdigest() #print("MD5 from " + name + ".bpa: " + new_md5) if md5_old == new_md5: - print("MD5 Checksum verified") + print("MD5 Checksum verified.") else: if ignore_corrupt == True: - print("WARNING: MD5 Checksum failed") + print("WARNING: MD5 Checksum failed!") else: - print("ERROR: MD5 Checksum failed") + print("ERROR: MD5 Checksum failed!") print("MD5 from " + md5file + ":", md5_old) print("MD5 from " + name + ".bpa: " + new_md5) - print("set flag --ignorecorrupt to still use corrupted data") + print("Set flag --ignorecorrupt to still use corrupted data.") exit() diff --git a/tests/CreateHDF5/test_CreateHDF5.py b/tests/CreateHDF5/test_CreateHDF5.py index a416f20..4589c7e 100644 --- a/tests/CreateHDF5/test_CreateHDF5.py +++ b/tests/CreateHDF5/test_CreateHDF5.py @@ -4,11 +4,12 @@ import subprocess import sys import warnings +import shutil import numpy as np -def test_bpcreatehdf5(): +def test_CreateHDF5(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -18,17 +19,32 @@ def test_bpcreatehdf5(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # Remove anything from previous tests + if (path / "BP_CreateHDF5").exists(): + shutil.rmtree(path / "BP_CreateHDF5") + if (path / ".BP_CreateHDF5").exists(): + os.remove(path / ".BP_CreateHDF5") + if (path / ".BP_CreateHDF5_BPL").exists(): + os.remove(path / ".BP_CreateHDF5_BPL") + if (path / "BP_CreateHDF5.bpa").exists(): + os.remove(path / "BP_CreateHDF5.bpa") + if (path / "BP_CreateHDF5.md5").exists(): + os.remove(path / "BP_CreateHDF5.md5") + # Run vspace - if not (path / "BP_CreateHDF5").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace.") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_CreateHDF5").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running MultiPlanet.") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / ".BP_CreateHDF5_BPL").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + print("Running BigPlanet.") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) file = path / "BP_CreateHDF5.bpa" @@ -37,4 +53,4 @@ def test_bpcreatehdf5(): if __name__ == "__main__": - test_bpcreatehdf5() + test_CreateHDF5() diff --git a/tests/ExtractArchive/test_ExtractArchive.py b/tests/ExtractArchive/test_ExtractArchive.py index 8bcbcf8..f69a101 100644 --- a/tests/ExtractArchive/test_ExtractArchive.py +++ b/tests/ExtractArchive/test_ExtractArchive.py @@ -4,14 +4,12 @@ import subprocess import sys import warnings - -import h5py +import shutil import numpy as np - import bigplanet as bp -def test_bpextract(): +def test_ExtractArchive(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -21,17 +19,32 @@ def test_bpextract(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # Remove anything from previous tests + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / ".BP_Extract_BPL").exists(): + os.remove(path / ".BP_Extract_BPL") + if (path / "BP_Extract.bpa").exists(): + os.remove(path / "BP_Extract.bpa") + if (path / "BP_Extract.md5").exists(): + os.remove(path / "BP_Extract.md5") + # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace.") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running MultiPlanet.") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / ".BP_Extract_BPL").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + print("Running BigPlanet.") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) file = bp.BPLFile(path / "BP_Extract.bpa") @@ -47,4 +60,4 @@ def test_bpextract(): if __name__ == "__main__": - test_bpextract() + test_ExtractArchive() diff --git a/tests/ExtractFilterArchive/test_ExtractFilterArchive.py b/tests/ExtractFilterArchive/test_ExtractFilterArchive.py index 8c30136..d544a7d 100644 --- a/tests/ExtractFilterArchive/test_ExtractFilterArchive.py +++ b/tests/ExtractFilterArchive/test_ExtractFilterArchive.py @@ -4,14 +4,12 @@ import subprocess import sys import warnings - -import h5py +import shutil import numpy as np - import bigplanet as bp -def test_bpextract(): +def test_ExtractFilterArchive(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -21,21 +19,37 @@ def test_bpextract(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # Remove any old test files/dirs + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / ".BP_Extract_BPL").exists(): + os.remove(path / ".BP_Extract_BPL") + if (path / "BP_Extract.bpa").exists(): + os.remove(path / "BP_Extract.bpa") + if (path / "Test.bpf").exists(): + os.remove(path / "Test.bpf") + # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace.") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running MultiPlanet.") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / "BP_Extract.bpa").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + print("Creating BigPlanet archive.") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) # Run bigplanet - if not (path / "Test.bpf").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + print("Creating BigPlanet file.") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) file = bp.BPLFile(path / "Test.bpf") @@ -57,4 +71,4 @@ def test_bpextract(): if __name__ == "__main__": - test_bpextract() + test_ExtractFilterArchive() diff --git a/tests/ExtractFilterRaw/test_ExtractFilterRaw.py b/tests/ExtractFilterRaw/test_ExtractFilterRaw.py index cc19865..9471ab2 100644 --- a/tests/ExtractFilterRaw/test_ExtractFilterRaw.py +++ b/tests/ExtractFilterRaw/test_ExtractFilterRaw.py @@ -11,7 +11,7 @@ import bigplanet as bp -def test_bpextract(): +def test_ExtractFilterRaw(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -23,20 +23,25 @@ def test_bpextract(): else: # If present, remove files from previous run if (path / "BP_Extract").exists(): - print("Removing BP_Extract") shutil.rmtree(path / "BP_Extract") if (path / ".BP_Extract").exists(): - print("Removing .BP_Extract") os.remove(path / ".BP_Extract") if (path / "Test.bpf").exists(): - print("Removing Test.bpf") os.remove(path / "Test.bpf") + # Run vspace print("Running vspace") + sys.stdout.flush() subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multiplanet print("Running multiplanet") + sys.stdout.flush() subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run BigPlanet print("Running bigplanet") + sys.stdout.flush() subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) file = bp.BPLFile(path / "Test.bpf") @@ -60,4 +65,4 @@ def test_bpextract(): if __name__ == "__main__": - test_bpextract() + test_ExtractFilterRaw() diff --git a/tests/MD5CheckSum/test_MD5CheckSum.py b/tests/MD5CheckSum/test_MD5CheckSum.py index 4f75b92..ba4a9bf 100644 --- a/tests/MD5CheckSum/test_MD5CheckSum.py +++ b/tests/MD5CheckSum/test_MD5CheckSum.py @@ -5,14 +5,9 @@ import subprocess import sys import warnings +import shutil -import h5py -import numpy as np - -import bigplanet as bp - - -def test_bpextract(): +def test_MD5CheckSum(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -22,17 +17,30 @@ def test_bpextract(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / ".BP_Extract_BPL").exists(): + os.remove(path / ".BP_Extract_BPL") + if (path / "BP_Extract.md5").exists(): + os.remove(path / "BP_Extract.md5") + # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / ".BP_Extract_BPL").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) bpa = path / "BP_Extract.bpa" @@ -49,4 +57,4 @@ def test_bpextract(): if __name__ == "__main__": - test_bpextract() + test_MD5CheckSum() diff --git a/tests/SingleSim/test_singlesim.py b/tests/SingleSim/test_singlesim.py index 11dc65a..91abfba 100644 --- a/tests/SingleSim/test_singlesim.py +++ b/tests/SingleSim/test_singlesim.py @@ -4,14 +4,11 @@ import subprocess import sys import warnings - -import h5py +import shutil import numpy as np - import bigplanet as bp - -def test_singlesim(): +def test_SingleSim(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -21,17 +18,30 @@ def test_singlesim(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / ".BP_Extract_BPL").exists(): + os.remove(path / ".BP_Extract_BPL") + if (path / "BP_Extract.md5").exists(): + os.remove(path / "BP_Extract.md5") + # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / "Test.bpf").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) file = bp.BPLFile(path / "Test.bpf") @@ -43,4 +53,4 @@ def test_singlesim(): if __name__ == "__main__": - test_singlesim() + test_SingleSim() diff --git a/tests/Stats/test_Stats.py b/tests/Stats/test_Stats.py index 9525c39..f049a17 100644 --- a/tests/Stats/test_Stats.py +++ b/tests/Stats/test_Stats.py @@ -4,14 +4,11 @@ import subprocess import sys import warnings - -import h5py +import shutil import numpy as np - import bigplanet as bp - -def test_bpstats(): +def test_Stats(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -21,17 +18,32 @@ def test_bpstats(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # If present, remove files from previous run + if (path / "BP_Stats").exists(): + shutil.rmtree(path / "BP_Stats") + if (path / ".BP_Stats").exists(): + os.remove(path / ".BP_Stats") + if (path / ".BP_Stats_BPL").exists(): + os.remove(path / ".BP_Stats_BPL") + if (path / "BP_Stats.md5").exists(): + os.remove(path / "BP_Stats.md5") + if (path / "BP_Stats.bpa").exists(): + os.remove(path / "BP_Stats.bpa") + # Run vspace - if not (path / "BP_Stats").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_Stats").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running MultiPlanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / "BP_Stats.bpa").exists(): - subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) + print("Running BigPlanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in", "-a"], cwd=path) file = bp.BPLFile(path / "BP_Stats.bpa") @@ -51,4 +63,4 @@ def test_bpstats(): if __name__ == "__main__": - test_bpstats() + test_Stats() diff --git a/tests/UlyssesAggregated/test_UlyssesAggregated.py b/tests/UlyssesAggregated/test_UlyssesAggregated.py index 30a3645..0689416 100644 --- a/tests/UlyssesAggregated/test_UlyssesAggregated.py +++ b/tests/UlyssesAggregated/test_UlyssesAggregated.py @@ -5,13 +5,12 @@ import subprocess import sys import warnings - +import shutil import numpy as np - import bigplanet.bp_extract as bp -def test_ulyssesaggregated(): +def test_UlyssesAggregated(): # gets current path path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) @@ -21,17 +20,28 @@ def test_ulyssesaggregated(): if cores == 1: warnings.warn("There is only 1 core on the machine", stacklevel=3) else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / "User.csv").exists(): + os.remove(path / "User.csv") + # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) # Run bigplanet - if not (path / "User.csv").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) file = path / "User.csv" @@ -42,4 +52,4 @@ def test_ulyssesaggregated(): if __name__ == "__main__": - test_ulyssesaggregated() + test_UlyssesAggregated() diff --git a/tests/UlyssesForward/test_UlyssesForward.py b/tests/UlyssesForward/test_UlyssesForward.py new file mode 100644 index 0000000..ba8498b --- /dev/null +++ b/tests/UlyssesForward/test_UlyssesForward.py @@ -0,0 +1,54 @@ +import multiprocessing as mp +import os +import pathlib +import subprocess +import sys +import warnings +import shutil +import numpy as np +import bigplanet as bp + +def test_UlyssesForward(): + # gets current path + path = pathlib.Path(__file__).parents[0].absolute() + sys.path.insert(1, str(path.parents[0])) + + # gets the number of cores on the machine + cores = mp.cpu_count() + if cores == 1: + warnings.warn("There is only 1 core on the machine", stacklevel=3) + else: + # If present, remove files from previous run + if (path / "BP_Extract").exists(): + shutil.rmtree(path / "BP_Extract") + if (path / ".BP_Extract").exists(): + os.remove(path / ".BP_Extract") + if (path / "User.csv").exists(): + os.remove(path / "User.csv") + + # Run vspace + print("Running vspace") + sys.stdout.flush() + subprocess.check_output(["vspace", "vspace.in"], cwd=path) + + # Run multi-planet + print("Running multiplanet") + sys.stdout.flush() + subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) + + # Run bigplanet + print("Running bigplanet") + sys.stdout.flush() + subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) + + file = path / "User.csv" + + data = bp.CSVToDict(file, 1) + + value = float(data["earth:TCore:forward"][451]) + + assert np.isclose(value, 4999.131849) + + +if __name__ == "__main__": + test_UlyssesForward() diff --git a/tests/UlyssesForward/test_bpulyssessforward.py b/tests/UlyssesForward/test_bpulyssessforward.py deleted file mode 100644 index e4e1ad9..0000000 --- a/tests/UlyssesForward/test_bpulyssessforward.py +++ /dev/null @@ -1,46 +0,0 @@ -import multiprocessing as mp -import os -import pathlib -import subprocess -import sys -import warnings - -import h5py -import numpy as np - -import bigplanet as bp - - -def test_ulyssesforward(): - # gets current path - path = pathlib.Path(__file__).parents[0].absolute() - sys.path.insert(1, str(path.parents[0])) - - # gets the number of cores on the machine - cores = mp.cpu_count() - if cores == 1: - warnings.warn("There is only 1 core on the machine", stacklevel=3) - else: - # Run vspace - if not (path / "BP_Extract").exists(): - subprocess.check_output(["vspace", "vspace.in"], cwd=path) - - # Run multi-planet - if not (path / ".BP_Extract").exists(): - subprocess.check_output(["multiplanet", "vspace.in"], cwd=path) - - # Run bigplanet - if not (path / "User.csv").exists(): - subprocess.check_output(["bigplanet", "bpl.in"], cwd=path) - - file = path / "User.csv" - - data = bp.CSVToDict(file, 1) - - value = float(data["earth:TCore:forward"][451]) - - assert np.isclose(value, 4999.131849) - - -if __name__ == "__main__": - test_ulyssesforward() From 4c471ae6785413da83488e8a21b43c9f5bfe6609 Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Wed, 26 Apr 2023 10:16:48 -0700 Subject: [PATCH 28/30] Added .gitignore to tests directory. --- tests/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/.gitignore diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..38a3510 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,2 @@ +.* +BP_* \ No newline at end of file From 2c35fd0eaa715ec969c12af6ca93c69f3f872ddf Mon Sep 17 00:00:00 2001 From: RoryBarnes Date: Wed, 26 Apr 2023 12:03:28 -0700 Subject: [PATCH 29/30] Added error handling if vplanet help doesn't work. Fixed issue with checking md5 files when running tests. --- bigplanet/bp_extract.py | 1 - bigplanet/bp_get.py | 17 +++++++++++------ tests/BP_Extract.md5 | 1 - tests/ExtractArchive/test_ExtractArchive.py | 2 ++ .../test_ExtractFilterArchive.py | 4 ++++ tests/Stats/test_Stats.py | 11 +++++++++-- 6 files changed, 26 insertions(+), 10 deletions(-) delete mode 100644 tests/BP_Extract.md5 diff --git a/bigplanet/bp_extract.py b/bigplanet/bp_extract.py index ca74887..4adf819 100755 --- a/bigplanet/bp_extract.py +++ b/bigplanet/bp_extract.py @@ -529,7 +529,6 @@ def Md5CheckSum(archivefile, ignore_corrupt=False): name = os.path.basename(archivefile.name).split(".")[0] bpa = archivefile - #print("Name: "+name) md5file = name + ".md5" # if it doesn't exist, we need to create it if os.path.isfile(md5file) == False: diff --git a/bigplanet/bp_get.py b/bigplanet/bp_get.py index 4b498fe..8587593 100755 --- a/bigplanet/bp_get.py +++ b/bigplanet/bp_get.py @@ -252,33 +252,38 @@ def GetVplanetHelp(): ): option = line.strip("** ") - # print("Option:",option) + #print("Option: ",option,repr(count)) vplanet_dict[option] = {} num = count + 1 while num != count: + #print("num,count: ",repr(num),repr(len(output))) + #print("Output[num]: ",repr(output[num])) + if (num >= len(output)): + print("ERROR: Unable to gather VPLanet help information!") + assert(0) if "Type" in output[num]: tp = output[num].rpartition("|")[-1].strip() vplanet_dict[option]["Type"] = tp - # print("Type:",tp) + #print("Type:",tp) num += 1 elif "Custom unit" in output[num]: custom_unit = output[num].rpartition("|")[-1].strip() vplanet_dict[option]["Custom Unit"] = custom_unit - # print("Custom Unit:",custom_unit) + #print("Custom Unit:",custom_unit) num += 1 elif "Dimension(s)" in output[num]: dim = output[num].rpartition("|")[-1].strip() vplanet_dict[option]["Dimension"] = dim - # print("Dimension:",dim) + #print("Dimension:",dim) num += 1 elif "Default value" in output[num]: default = output[num].rpartition("|")[-1].strip() vplanet_dict[option]["Default Value"] = default - # print("Default Value",default) - # print() + #print("Default Value",default) + #print() num += 1 diff --git a/tests/BP_Extract.md5 b/tests/BP_Extract.md5 deleted file mode 100644 index 369e98a..0000000 --- a/tests/BP_Extract.md5 +++ /dev/null @@ -1 +0,0 @@ -bb3e5ea8fe70c0e2a9f7e34798400168 \ No newline at end of file diff --git a/tests/ExtractArchive/test_ExtractArchive.py b/tests/ExtractArchive/test_ExtractArchive.py index f69a101..b03927d 100644 --- a/tests/ExtractArchive/test_ExtractArchive.py +++ b/tests/ExtractArchive/test_ExtractArchive.py @@ -28,6 +28,8 @@ def test_ExtractArchive(): os.remove(path / ".BP_Extract_BPL") if (path / "BP_Extract.bpa").exists(): os.remove(path / "BP_Extract.bpa") + if (path / "../BP_Extract.md5").exists(): + os.remove(path / "../BP_Extract.md5") if (path / "BP_Extract.md5").exists(): os.remove(path / "BP_Extract.md5") diff --git a/tests/ExtractFilterArchive/test_ExtractFilterArchive.py b/tests/ExtractFilterArchive/test_ExtractFilterArchive.py index d544a7d..bd390f4 100644 --- a/tests/ExtractFilterArchive/test_ExtractFilterArchive.py +++ b/tests/ExtractFilterArchive/test_ExtractFilterArchive.py @@ -30,6 +30,10 @@ def test_ExtractFilterArchive(): os.remove(path / "BP_Extract.bpa") if (path / "Test.bpf").exists(): os.remove(path / "Test.bpf") + if (path / "../BP_Extract.md5").exists(): + os.remove(path / "../BP_Extract.md5") + if (path / "BP_Extract.md5").exists(): + os.remove(path / "BP_Extract.md5") # Run vspace print("Running vspace.") diff --git a/tests/Stats/test_Stats.py b/tests/Stats/test_Stats.py index f049a17..91588fb 100644 --- a/tests/Stats/test_Stats.py +++ b/tests/Stats/test_Stats.py @@ -13,6 +13,11 @@ def test_Stats(): path = pathlib.Path(__file__).parents[0].absolute() sys.path.insert(1, str(path.parents[0])) + # up = os.path.normpath("../") + # if (up / "*.md5").exists: + # os.remove(path / "*.md5") + + # gets the number of cores on the machine cores = mp.cpu_count() if cores == 1: @@ -25,10 +30,12 @@ def test_Stats(): os.remove(path / ".BP_Stats") if (path / ".BP_Stats_BPL").exists(): os.remove(path / ".BP_Stats_BPL") - if (path / "BP_Stats.md5").exists(): - os.remove(path / "BP_Stats.md5") if (path / "BP_Stats.bpa").exists(): os.remove(path / "BP_Stats.bpa") + if (path / "../BP_Stats.md5").exists(): + os.remove(path / "../BP_Stats.md5") + if (path / "BP_Stats.md5").exists(): + os.remove(path / "BP_Stats.md5") # Run vspace print("Running vspace") From f1caa7a31f0d1ac0bf3315da91640e956e32bfb2 Mon Sep 17 00:00:00 2001 From: Rory Barnes Date: Wed, 23 Aug 2023 18:44:15 -0700 Subject: [PATCH 30/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5623404..b0333f0 100644 --- a/README.md +++ b/README.md @@ -17,4 +17,4 @@

-A suite of data analysis and extraction routines for [VPLanet](https://github.com/VirtualPlanetaryLaboratory/vplanet) and designed to be used with [VSPACE](https://github.com/VirtualPlanetaryLaboratory/vspace) and [MultiPlanet](https://github.com/VirtualPlanetaryLaboratory/multi-planet) scripts. It can also generate files for use with the free version of the [VR Ulysses data explorer](https://www.vrulysses.com/download-ulysses). [Read the docs](https://VirtualPlanetaryLaboratory.github.io/bigplanet/) to learn how to get the most out of your VPLanet parameter sweeps. +A suite of data analysis and extraction routines for [VPLanet](https://github.com/VirtualPlanetaryLaboratory/vplanet) and designed to be used with [VSPACE](https://github.com/VirtualPlanetaryLaboratory/vspace) and [MultiPlanet](https://github.com/VirtualPlanetaryLaboratory/multi-planet) scripts. It can also generate files for use with the free version of the [VR Ulysses data explorer](https://www.vrulysses.com/download-ulysses). [Read the docs](https://VirtualPlanetaryLaboratory.github.io/bigplanet/) to learn how to get the most out of your `VPLanet` parameter sweeps.