From 32efcc25d2a0259ccf7897f290e12ea5ed785760 Mon Sep 17 00:00:00 2001 From: evacougnon Date: Fri, 31 Jul 2026 10:26:35 +1000 Subject: [PATCH] add sattag dm config only --- ..._relay_tagging_location_qc_delayed_qc.json | 307 +++++++++ ..._relay_tagging_location_qc_delayed_qc.json | 291 +++++++++ ..._relay_tagging_location_qc_delayed_qc.json | 609 ++++++++++++++++++ ..._relay_tagging_location_qc_delayed_qc.json | 278 ++++++++ ..._relay_tagging_location_qc_delayed_qc.json | 240 +++++++ ..._relay_tagging_location_qc_delayed_qc.json | 255 ++++++++ ..._relay_tagging_location_qc_delayed_qc.json | 206 ++++++ ..._relay_tagging_location_qc_delayed_qc.json | 367 +++++++++++ 8 files changed, 2553 insertions(+) create mode 100644 aodn_cloud_optimised/config/dataset/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.json create mode 100644 aodn_cloud_optimised/config/dataset/animal_diag_satellite_relay_tagging_location_qc_delayed_qc.json create mode 100644 aodn_cloud_optimised/config/dataset/animal_dive_satellite_relay_tagging_location_qc_delayed_qc.json create mode 100644 aodn_cloud_optimised/config/dataset/animal_gps_satellite_relay_tagging_location_qc_delayed_qc.json create mode 100644 aodn_cloud_optimised/config/dataset/animal_haulout_satellite_relay_tagging_location_qc_delayed_qc.json create mode 100644 aodn_cloud_optimised/config/dataset/animal_metadata_satellite_relay_tagging_location_qc_delayed_qc.json create mode 100644 aodn_cloud_optimised/config/dataset/animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc.json create mode 100644 aodn_cloud_optimised/config/dataset/animal_summary_satellite_relay_tagging_location_qc_delayed_qc.json diff --git a/aodn_cloud_optimised/config/dataset/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.json b/aodn_cloud_optimised/config/dataset/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.json new file mode 100644 index 00000000..31d8c9f8 --- /dev/null +++ b/aodn_cloud_optimised/config/dataset/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.json @@ -0,0 +1,307 @@ +{ + "dataset_name": "animal_ctd_satellite_relay_tagging_location_qc_delayed_qc", + "logger_name": "animal_ctd_satellite_relay_tagging_location_qc_delayed_qc", + "cloud_optimised_format": "parquet", + "run_settings": { + "paths": [ + { + "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM", + "filter": [ + ".*_ctd_.*_dm\\.csv$" + ], + "year_range": [] + } + ], + "cluster": { + "mode": "local", + "restart_every_path": false + }, + "clear_existing_data": true, + "raise_error": false, + "coiled_cluster_options": { + "n_workers": [ + 1, + 20 + ], + "scheduler_vm_types": "m7i-flex.large", + "worker_vm_types": "m7i-flex.large", + "allow_ingress_from": "me", + "compute_purchase_option": "spot_with_fallback", + "worker_options": { + "nthreads": 4, + "memory_limit": "8GB" + } + }, + "batch_size": 5, + "force_previous_parquet_deletion": false + }, + "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5", + "schema": { + "ref": { + "type": "string", + "long_name": "reference", + "comment": "A three-part code (e.g. rs3-Fred-07) : The first part identifies the batch of tags, the second part is the name or number of the animal, the third part is the year" + }, + "ptt": { + "type": "int32", + "long_name": "Platform Transmitter Terminal", + "comment": "Argos PTT number is used" + }, + "end_date": { + "type": "timestamp[ns]", + "comment": "The time (UTC) of the end of the upcast" + }, + "max_dbar": { + "type": "double", + "long_name": "Maximum pressure", + "comment": "The maximum pressure (in decibar, approximately 1 metre)" + }, + "num": { + "type": "int32", + "long_name": "Upcast number", + "comment": "Upcasts are assigned sequential numbers as they are performed. The SRDL transmits only the deepest casts that occur in each period (usually 6 hours or 12 hours). This means that some earlier upcasts will not be transmitted so these numbers will not be consecutive." + }, + "n_temp": { + "type": "int32", + "comment": "The number of (pressure,temperature) pairs " + }, + "n_cond": { + "type": "double", + "comment": "The number of (pressure,conductivity) pairs " + }, + "n_sal": { + "type": "int32", + "comment": "The number of (pressure,salinity) pairs " + }, + "temp_dbar": { + "type": "double", + "long_name": "Pressure coordinates of the (pressure,temperature) profile", + "units": "decibars", + "comment": "A comma-separated list of the pressure coordinates of the (pressure,temperature) profile (decibars)" + }, + "temp_vals": { + "type": "double", + "long_name": "Temperature profile", + "units": "degrees Celsius", + "comment": "A comma-separated list of the temperature coordinates of the (pressure,temperature) profile (degree C)" + }, + "cond_dbar": { + "type": "double", + "long_name": "Pressure coordinates of the (pressure,conductivity) profile", + "units": "decibars", + "comment": "A comma-separated list of the pressure coordinates of the (pressure,conductivity) profile (decibars)" + }, + "cond_vals": { + "type": "double", + "long_name": "Conductivity profile", + "units": "mS/cm", + "comment": "A comma-separated list of the conductivity coordinates of the (pressure,conductivity) profile (mS/cm)" + }, + "sal_dbar": { + "type": "double", + "long_name": "Pressure coordinates of the (pressure,salinity) profile", + "units": "decibars", + "comment": "A comma-separated list of the pressure coordinates of the (pressure,salinity) profile (decibars)" + }, + "sal_vals": { + "type": "double", + "long_name": "Salinity profile", + "units": " ", + "comment": "A comma-separated list of the salinity coordinates of the (pressure,salinity) profile" + }, + "n_fluoro": { + "type": "double" + }, + "fluoro_dbar": { + "type": "double", + "units": "decibars" + }, + "fluoro_vals": { + "type": "double" + }, + "n_oxy": { + "type": "double" + }, + "oxy_dbar": { + "type": "double", + "units": "decibars" + }, + "oxy_vals": { + "type": "double" + }, + "qc_profile": { + "type": "int32" + }, + "qc_temp": { + "type": "double" + }, + "qc_sal": { + "type": "double" + }, + "sal_corrected_vals": { + "type": "double" + }, + "created": { + "type": "string", + "long_name": "Date of file creation (UTC)" + }, + "modified": { + "type": "string", + "long_name": "Date of update of this file (UTC)" + }, + "n_photo": { + "type": "double" + }, + "photo_dbar": { + "type": "double" + }, + "photo_vals": { + "type": "double" + }, + "lat": { + "type": "double", + "long_name": "Latitude", + "units": "degrees_north", + "comment": "Position estimate (WGS 84 datum)" + }, + "lon": { + "type": "double", + "long_name": "Longitude", + "units": "degrees_east", + "comment": "Position estimate (WGS 84 datum)" + }, + "ssm_lon": { + "type": "double" + }, + "ssm_lat": { + "type": "double" + }, + "ssm_x": { + "type": "double" + }, + "ssm_y": { + "type": "double" + }, + "ssm_x_se": { + "type": "double" + }, + "ssm_y_se": { + "type": "double" + }, + "cid": { + "type": "string", + "comment": "SMRU campaign name " + } + }, + "aws_opendata_registry": { + "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control", + "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5", + "Contact": "info@aodn.org.au", + "ManagedBy": "AODN", + "UpdateFrequency": "As Needed", + "Tags": [ + "FILL UP MANUALLY - CHECK DOCUMENTATION" + ], + "License": "http://creativecommons.org/licenses/by/4.0/", + "Resources": [ + { + "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.parquet", + "Region": "ap-southeast-2", + "Type": "S3 Bucket" + } + ], + "DataAtWork": { + "Tutorials": [ + { + "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + }, + { + "Title": "Accessing and search for any AODN dataset", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + } + ] + }, + "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" + }, + "csv_config": { + "pandas_read_csv_config": { + "na_values": [ + "N/A", + "NaN" + ], + "encoding": "utf-8", + "index_col": "end_date", + "parse_dates": [ + "end_date" + ] + } + }, + "schema_transformation": { + "drop_variables": [], + "add_variables": { + "filename": { + "source": "@filename", + "schema": { + "type": "string", + "units": "1", + "long_name": "Filename of the source file" + } + }, + "timestamp": { + "source": "@partitioning:time_extent", + "schema": { + "type": "int64", + "units": "1", + "long_name": "Partition timestamp" + } + }, + "polygon": { + "source": "@partitioning:spatial_extent", + "schema": { + "type": "string", + "units": "1", + "long_name": "Spatial partition polygon" + } + } + }, + "partitioning": [ + { + "source_variable": "timestamp", + "type": "time_extent", + "time_extent": { + "time_varname": "end_date", + "partition_period": "Y" + } + }, + { + "source_variable": "polygon", + "type": "spatial_extent", + "spatial_extent": { + "lat_varname": "lat", + "lon_varname": "lon", + "spatial_resolution": 20 + } + } + ], + "global_attributes": { + "delete": [ + "geospatial_lat_max", + "geospatial_lat_min", + "geospatial_lon_max", + "geospatial_lon_min", + "date_created" + ], + "set": {} + } + }, + "handler_class": "AatamsSatelliteTaggingHandler" +} diff --git a/aodn_cloud_optimised/config/dataset/animal_diag_satellite_relay_tagging_location_qc_delayed_qc.json b/aodn_cloud_optimised/config/dataset/animal_diag_satellite_relay_tagging_location_qc_delayed_qc.json new file mode 100644 index 00000000..e4ca3cce --- /dev/null +++ b/aodn_cloud_optimised/config/dataset/animal_diag_satellite_relay_tagging_location_qc_delayed_qc.json @@ -0,0 +1,291 @@ +{ + "dataset_name": "animal_diag_satellite_relay_tagging_location_qc_delayed_qc", + "logger_name": "animal_diag_satellite_relay_tagging_location_qc_delayed_qc", + "cloud_optimised_format": "parquet", + "run_settings": { + "paths": [ + { + "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM", + "filter": [ + ".*_diag_.*_dm\\.csv$" + ], + "year_range": [] + } + ], + "cluster": { + "mode": "local", + "restart_every_path": false + }, + "clear_existing_data": true, + "raise_error": false, + "coiled_cluster_options": { + "n_workers": [ + 1, + 20 + ], + "scheduler_vm_types": "m7i-flex.large", + "worker_vm_types": "m7i-flex.large", + "allow_ingress_from": "me", + "compute_purchase_option": "spot_with_fallback", + "worker_options": { + "nthreads": 4, + "memory_limit": "8GB" + } + }, + "batch_size": 5, + "force_previous_parquet_deletion": false + }, + "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5", + "schema": { + "ref": { + "type": "string", + "long_name": "reference", + "comment": "A three-part code (e.g. rs3-Fred-07) : The first part identifies the batch of tags, the second part is the name or number of the animal, the third part is the year" + }, + "ptt": { + "type": "int32", + "long_name": "Platform Transmitter Terminal", + "comment": "Argos PTT number is used" + }, + "d_date": { + "type": "timestamp[ns]", + "comment": "The time assigned to the location by ARGOS (UTC)" + }, + "lq": { + "type": "int32", + "comment": "The Location Quality assigned by Argos (-1 = class A, -2 = class B, -9 = class Z)" + }, + "lat": { + "type": "double", + "long_name": "Latitude", + "units": "degrees_north", + "comment": "Position estimate (WGS 84 datum)" + }, + "lon": { + "type": "double", + "long_name": "Longitude", + "units": "degrees_east", + "comment": "Position estimate (WGS 84 datum)" + }, + "alt_lat": { + "type": "double", + "comment": "Alternative solution to position equations (this may have been swapped with LAT,LON by the location filtering algorithm). These fields are simply copies of LAT,LON when Kalman location processing is operating." + }, + "alt_lon": { + "type": "double", + "comment": "Alternative solution to position equations (this may have been swapped with LAT,LON by the location filtering algorithm). These fields are simply copies of LAT,LON when Kalman location processing is operating." + }, + "n_mess": { + "type": "int32", + "comment": "Number of uplinks received during the satellite pass" + }, + "n_mess_120": { + "type": "int32", + "comment": "Number of uplinks received with signal strength > -120 dB" + }, + "best_level": { + "type": "int32", + "units": "dB", + "comment": "Signal strength of strongest uplink (dB)" + }, + "pass_dur": { + "type": "int32", + "units": "s", + "comment": "Duration of satellite overpass (s)" + }, + "freq": { + "type": "double", + "units": "Hz", + "comment": "Measured frequency of SRDL signal at the satellite (Hz)" + }, + "v_mask": { + "type": "int32", + "comment": "Result of the location filtering algorithm: 0 = accepted, 20 = rejected at the 20 dm/s level (see VMASK_THRESHOLD). The first and last two points of the track are excluded from the algorithm and are assigned -1" + }, + "alt": { + "type": "double", + "comment": "= 1 if the LAT,LON and ALT_LAT, ALT_LON were swapped by the location filter\u2019s initial plausibility check" + }, + "est_speed": { + "type": "double", + "long_name": "Estimated speed", + "units": "m/s", + "comment": "Speed across the surface estimated by the location filtering algorithm (m/s)" + }, + "km_from_home": { + "type": "double", + "long_name": "Distance from home", + "units": "km", + "comment": "Great-circle distance to the release_latitude, release_longitude position specified in the metadata table (km). This is simply the shortest distance between the two points: the path may cross land." + }, + "iq": { + "type": "int32", + "comment": "A two-digit code which gives further information about the location calculation for poor quality locations (see Argos documentation for a complete explanation)" + }, + "nops": { + "type": "int32" + }, + "deleted": { + "type": "string", + "comment": "The value Y here causes this position to be ignored by the location filtering algorithm. Normally used to remove a very poor position occurring at the beginning of the track" + }, + "actual_ptt": { + "type": "int32", + "comment": "When a single SRDL is programmed to use several different PTT numbers, the SMRU system overrides the PTT field to present all the data as if it had been transmitted under a single PTT number. This field records the true PTT number associated with this location." + }, + "error_radius": { + "type": "int32" + }, + "semi_major_axis": { + "type": "int32" + }, + "semi_minor_axis": { + "type": "int32" + }, + "ellipse_orientation": { + "type": "int32" + }, + "hdop": { + "type": "double" + }, + "satellite": { + "type": "double" + }, + "diag_id": { + "type": "int32" + }, + "ssm_lon": { + "type": "double" + }, + "ssm_lat": { + "type": "double" + }, + "ssm_x": { + "type": "double" + }, + "ssm_y": { + "type": "double" + }, + "ssm_x_se": { + "type": "double" + }, + "ssm_y_se": { + "type": "double" + }, + "cid": { + "type": "string", + "comment": "SMRU campaign name " + } + }, + "aws_opendata_registry": { + "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control", + "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5", + "Contact": "info@aodn.org.au", + "ManagedBy": "AODN", + "UpdateFrequency": "As Needed", + "Tags": [ + "FILL UP MANUALLY - CHECK DOCUMENTATION" + ], + "License": "http://creativecommons.org/licenses/by/4.0/", + "Resources": [ + { + "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_diag_satellite_relay_tagging_location_qc_delayed_qc.parquet", + "Region": "ap-southeast-2", + "Type": "S3 Bucket" + } + ], + "DataAtWork": { + "Tutorials": [ + { + "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_diag_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_diag_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + }, + { + "Title": "Accessing and search for any AODN dataset", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + } + ] + }, + "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" + }, + "csv_config": { + "pandas_read_csv_config": { + "na_values": [ + "N/A", + "NaN" + ], + "encoding": "utf-8", + "index_col": "d_date", + "parse_dates": [ + "d_date" + ] + } + }, + "schema_transformation": { + "drop_variables": [], + "add_variables": { + "filename": { + "source": "@filename", + "schema": { + "type": "string", + "units": "1", + "long_name": "Filename of the source file" + } + }, + "timestamp": { + "source": "@partitioning:time_extent", + "schema": { + "type": "int64", + "units": "1", + "long_name": "Partition timestamp" + } + }, + "polygon": { + "source": "@partitioning:spatial_extent", + "schema": { + "type": "string", + "units": "1", + "long_name": "Spatial partition polygon" + } + } + }, + "partitioning": [ + { + "source_variable": "timestamp", + "type": "time_extent", + "time_extent": { + "time_varname": "d_date", + "partition_period": "Y" + } + }, + { + "source_variable": "polygon", + "type": "spatial_extent", + "spatial_extent": { + "lat_varname": "lat", + "lon_varname": "lon", + "spatial_resolution": 20 + } + } + ], + "global_attributes": { + "delete": [ + "geospatial_lat_max", + "geospatial_lat_min", + "geospatial_lon_max", + "geospatial_lon_min", + "date_created" + ], + "set": {} + } + }, + "handler_class": "AatamsSatelliteTaggingHandler" +} diff --git a/aodn_cloud_optimised/config/dataset/animal_dive_satellite_relay_tagging_location_qc_delayed_qc.json b/aodn_cloud_optimised/config/dataset/animal_dive_satellite_relay_tagging_location_qc_delayed_qc.json new file mode 100644 index 00000000..509c8290 --- /dev/null +++ b/aodn_cloud_optimised/config/dataset/animal_dive_satellite_relay_tagging_location_qc_delayed_qc.json @@ -0,0 +1,609 @@ +{ + "dataset_name": "animal_dive_satellite_relay_tagging_location_qc_delayed_qc", + "logger_name": "animal_dive_satellite_relay_tagging_location_qc_delayed_qc", + "cloud_optimised_format": "parquet", + "run_settings": { + "paths": [ + { + "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM", + "filter": [ + ".*_dive_.*_dm\\.csv$" + ], + "year_range": [] + } + ], + "cluster": { + "mode": "local", + "restart_every_path": false + }, + "clear_existing_data": true, + "raise_error": false, + "coiled_cluster_options": { + "n_workers": [ + 1, + 20 + ], + "scheduler_vm_types": "m7i-flex.large", + "worker_vm_types": "m7i-flex.large", + "allow_ingress_from": "me", + "compute_purchase_option": "spot_with_fallback", + "worker_options": { + "nthreads": 4, + "memory_limit": "8GB" + } + }, + "batch_size": 5, + "force_previous_parquet_deletion": false + }, + "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5", + "schema": { + "ref": { + "type": "string", + "long_name": "reference", + "comment": "A three-part code (e.g. rs3-Fred-07) : The first part identifies the batch of tags, the second part is the name or number of the animal, the third part is the year" + }, + "ptt": { + "type": "int32", + "long_name": "Platform Transmitter Terminal", + "comment": "Argos PTT number is used" + }, + "cnt": { + "type": "int32", + "comment": "The number of times that this record was received" + }, + "de_date": { + "type": "timestamp[ns]", + "long_name": "Dive end date", + "comment": "Dive end: the time (UTC) at which the depth crossed the threshold on the return to the surface" + }, + "surf_dur": { + "type": "int32", + "long_name": "Surface duration", + "units": "s", + "comment": "Surface duration (seconds): the length of time following DE_DATE before the next dive or other event began." + }, + "dive_dur": { + "type": "int32", + "long_name": "Dive duration", + "units": "s", + "comment": "Dive duration (seconds): the length of time between the crossing of the dive-start threshold on the descent and DE_DATE" + }, + "max_dep": { + "type": "double", + "units": "m", + "comment": "The maximum dive depth recorded" + }, + "d1": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d2": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d3": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d4": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "v1": { + "type": "double", + "units": "m/s", + "comment": "Speeds (metres/second) in each section of the dive. If there is one more Vvalue than D- values then V1 is the speed up to D1, V2 between D1 and D2, etc. The last V- value is the speed between the last D- value and the surface. If there are three V- values then V1 is the speed from the surface to the D1 point, V3 is the speed from the last depth point to the surface, and V2 is the speed over the remainder of the dive between D1 and the last depth point." + }, + "v2": { + "type": "double", + "units": "m/s", + "comment": "Speeds (metres/second) in each section of the dive. If there is one more Vvalue than D- values then V1 is the speed up to D1, V2 between D1 and D2, etc. The last V- value is the speed between the last D- value and the surface. If there are three V- values then V1 is the speed from the surface to the D1 point, V3 is the speed from the last depth point to the surface, and V2 is the speed over the remainder of the dive between D1 and the last depth point." + }, + "v3": { + "type": "double", + "units": "m/s", + "comment": "Speeds (metres/second) in each section of the dive. If there is one more Vvalue than D- values then V1 is the speed up to D1, V2 between D1 and D2, etc. The last V- value is the speed between the last D- value and the surface. If there are three V- values then V1 is the speed from the surface to the D1 point, V3 is the speed from the last depth point to the surface, and V2 is the speed over the remainder of the dive between D1 and the last depth point." + }, + "v4": { + "type": "double", + "units": "m/s", + "comment": "Speeds (metres/second) in each section of the dive. If there is one more Vvalue than D- values then V1 is the speed up to D1, V2 between D1 and D2, etc. The last V- value is the speed between the last D- value and the surface. If there are three V- values then V1 is the speed from the surface to the D1 point, V3 is the speed from the last depth point to the surface, and V2 is the speed over the remainder of the dive between D1 and the last depth point." + }, + "v5": { + "type": "double", + "units": "m/s", + "comment": "Speeds (metres/second) in each section of the dive. If there is one more Vvalue than D- values then V1 is the speed up to D1, V2 between D1 and D2, etc. The last V- value is the speed between the last D- value and the surface. If there are three V- values then V1 is the speed from the surface to the D1 point, V3 is the speed from the last depth point to the surface, and V2 is the speed over the remainder of the dive between D1 and the last depth point." + }, + "travel_r": { + "type": "double", + "long_name": "estimate of speed-over-ground", + "units": "m/s", + "comment": " An estimate of speed-over-ground, based only on filtered Argos locations not the turbine speed (metres/second)" + }, + "homedist": { + "type": "double", + "comment": "Great-circle distance from the home point in the metadata table" + }, + "bottom": { + "type": "double", + "units": "m", + "comment": "An estimate of the ocean depth derived from the Smith and Sandwell gridded dataset (metres)" + }, + "t1": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t2": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t3": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t4": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "d_speed": { + "type": "double", + "units": "m/s", + "comment": "The average speed recorded in the dive (metres/second)" + }, + "n_depths": { + "type": "int32", + "comment": "The number of intermediate dive points present" + }, + "n_speeds": { + "type": "int32", + "comment": "The number of intermediate speeds present" + }, + "depth_str": { + "type": "string", + "comment": "The D1,D2,D3, values as a comma-separated string" + }, + "speed_str": { + "type": "string", + "comment": "The V1,V2,V3, values as a comma-separated string" + }, + "propn_str": { + "type": "string", + "comment": "The T1,T2,T3, values as a comma-separated string" + }, + "percent_area": { + "type": "double", + "comment": "Used to calculate the TAD dive index" + }, + "residual": { + "type": "int32" + }, + "grp_number": { + "type": "int32", + "comment": "Shows which dives were sent in the same transmission" + }, + "d5": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "t5": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "degc_str": { + "type": "double" + }, + "illum_str": { + "type": "double" + }, + "pca_desc": { + "type": "double" + }, + "pca_btm": { + "type": "double" + }, + "pca_asc": { + "type": "double" + }, + "pca_max_desc": { + "type": "double" + }, + "pca_max_btm": { + "type": "double" + }, + "pca_max_asc": { + "type": "double" + }, + "pca_mean_desc": { + "type": "double" + }, + "pca_mean_btm": { + "type": "double" + }, + "pca_mean_asc": { + "type": "double" + }, + "swim_eff_desc": { + "type": "double" + }, + "swim_eff_btm": { + "type": "double" + }, + "swim_eff_asc": { + "type": "double" + }, + "swim_eff_whole": { + "type": "double" + }, + "secs_desc": { + "type": "double" + }, + "secs_btm": { + "type": "double" + }, + "secs_asc": { + "type": "double" + }, + "pitch_desc": { + "type": "double" + }, + "pitch_btm": { + "type": "double" + }, + "pitch_asc": { + "type": "double" + }, + "pitch_str": { + "type": "double" + }, + "tagging_id": { + "type": "double" + }, + "de_date_tag": { + "type": "string" + }, + "qc": { + "type": "double" + }, + "d6": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d7": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d8": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d9": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d10": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d11": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d12": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d13": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d14": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d15": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d16": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d17": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d18": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d19": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d20": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d21": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d22": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d23": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d24": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "d25": { + "type": "double", + "units": "m", + "comment": "Intermediate depth points (metres) in the dive. For most deployments one of these is guaranteed to be the maximum depth (so MAX_DEP is not transmitted)" + }, + "t6": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t7": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t8": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t9": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t10": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t11": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t12": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t13": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t14": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t15": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t16": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t17": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t18": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t19": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t20": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t21": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t22": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t23": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t24": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "t25": { + "type": "double", + "comment": "The percentage of DIVE_DUR elapsed since the beginning of the dive at each of the intermediate depth points" + }, + "ds_date": { + "type": "string" + }, + "start_lat": { + "type": "double", + "comment": "The approximate position of the SRDL at the start of the deployment" + }, + "start_lon": { + "type": "double", + "comment": "The approximate position of the SRDL at the start of the deployment" + }, + "lat": { + "type": "double", + "long_name": "Latitude", + "units": "degrees_north", + "comment": "Position estimate (WGS 84 datum)" + }, + "lon": { + "type": "double", + "long_name": "Longitude", + "units": "degrees_east", + "comment": "Position estimate (WGS 84 datum)" + }, + "ssm_lon": { + "type": "double" + }, + "ssm_lat": { + "type": "double" + }, + "ssm_x": { + "type": "double" + }, + "ssm_y": { + "type": "double" + }, + "ssm_x_se": { + "type": "double" + }, + "ssm_y_se": { + "type": "double" + }, + "cid": { + "type": "string", + "comment": "SMRU campaign name " + } + }, + "aws_opendata_registry": { + "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control", + "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5", + "Contact": "info@aodn.org.au", + "ManagedBy": "AODN", + "UpdateFrequency": "As Needed", + "Tags": [ + "FILL UP MANUALLY - CHECK DOCUMENTATION" + ], + "License": "http://creativecommons.org/licenses/by/4.0/", + "Resources": [ + { + "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_dive_satellite_relay_tagging_location_qc_delayed_qc.parquet", + "Region": "ap-southeast-2", + "Type": "S3 Bucket" + } + ], + "DataAtWork": { + "Tutorials": [ + { + "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_dive_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_dive_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + }, + { + "Title": "Accessing and search for any AODN dataset", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + } + ] + }, + "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" + }, + "csv_config": { + "pandas_read_csv_config": { + "na_values": [ + "N/A", + "NaN" + ], + "encoding": "utf-8", + "index_col": "de_date", + "parse_dates": [ + "de_date" + ] + } + }, + "schema_transformation": { + "drop_variables": [], + "add_variables": { + "filename": { + "source": "@filename", + "schema": { + "type": "string", + "units": "1", + "long_name": "Filename of the source file" + } + }, + "timestamp": { + "source": "@partitioning:time_extent", + "schema": { + "type": "int64", + "units": "1", + "long_name": "Partition timestamp" + } + }, + "polygon": { + "source": "@partitioning:spatial_extent", + "schema": { + "type": "string", + "units": "1", + "long_name": "Spatial partition polygon" + } + } + }, + "partitioning": [ + { + "source_variable": "timestamp", + "type": "time_extent", + "time_extent": { + "time_varname": "de_date", + "partition_period": "Y" + } + }, + { + "source_variable": "polygon", + "type": "spatial_extent", + "spatial_extent": { + "lat_varname": "lat", + "lon_varname": "lon", + "spatial_resolution": 20 + } + } + ], + "global_attributes": { + "delete": [ + "geospatial_lat_max", + "geospatial_lat_min", + "geospatial_lon_max", + "geospatial_lon_min", + "date_created" + ], + "set": {} + } + }, + "handler_class": "AatamsSatelliteTaggingHandler" +} diff --git a/aodn_cloud_optimised/config/dataset/animal_gps_satellite_relay_tagging_location_qc_delayed_qc.json b/aodn_cloud_optimised/config/dataset/animal_gps_satellite_relay_tagging_location_qc_delayed_qc.json new file mode 100644 index 00000000..86e7ad1f --- /dev/null +++ b/aodn_cloud_optimised/config/dataset/animal_gps_satellite_relay_tagging_location_qc_delayed_qc.json @@ -0,0 +1,278 @@ +{ + "dataset_name": "animal_gps_satellite_relay_tagging_location_qc_delayed_qc", + "logger_name": "animal_gps_satellite_relay_tagging_location_qc_delayed_qc", + "cloud_optimised_format": "parquet", + "run_settings": { + "paths": [ + { + "type": "files", + "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM", + "filter": [ + ".*_gps_.*_dm\\.csv$" + ], + "year_range": [] + } + ], + "cluster": { + "mode": "local", + "restart_every_path": false + }, + "clear_existing_data": true, + "raise_error": false, + "coiled_cluster_options": { + "n_workers": [ + 1, + 20 + ], + "scheduler_vm_types": "m7i-flex.large", + "worker_vm_types": "m7i-flex.large", + "allow_ingress_from": "me", + "compute_purchase_option": "spot_with_fallback", + "worker_options": { + "nthreads": 4, + "memory_limit": "8GB" + } + }, + "batch_size": 5, + "force_previous_parquet_deletion": false + }, + "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5", + "schema": { + "ref": { + "type": "string", + "long_name": "reference", + "comment": "A three-part code (e.g. rs3-Fred-07) : The first part identifies the batch of tags, the second part is the name or number of the animal, the third part is the year" + }, + "ptt": { + "type": "int32", + "long_name": "Platform Transmitter Terminal", + "comment": "Argos PTT number is used" + }, + "d_date": { + "type": "timestamp[ns]", + "comment": "The time that the fix was obtained (UTC)" + }, + "nsats_detected": { + "type": "int32", + "comment": "The number of SVs detected by the tag" + }, + "nsats_transmitted": { + "type": "int32", + "comment": "The number of ranges to SVs transmitted by the tag. SRDLs normally transmit only the 8 strongest signals when more than 8 are detected. Only 5 SVs are required to generate a location and error estimate." + }, + "gps_mode": { + "type": "int32" + }, + "pseudoranges": { + "type": "string" + }, + "max_csn": { + "type": "string", + "comment": "The carrier-to-noise ratio of the strongest SV" + }, + "cnt": { + "type": "int32", + "comment": "The number of times that this record was received" + }, + "lat": { + "type": "double", + "long_name": "Latitude", + "units": "degrees_north", + "comment": "Position estimate (WGS 84 datum)" + }, + "lon": { + "type": "double", + "long_name": "Longitude", + "units": "degrees_east", + "comment": "Position estimate (WGS 84 datum)" + }, + "residual": { + "type": "float32", + "comment": "A measure of the uncertainty of the location." + }, + "timeshift": { + "type": "float32", + "comment": "Estimated discrepancy between the tags real-time clock and the atomic clock maintained on board the GPS satellites. This value should drift steadily (about 0.1s/day)" + }, + "submitted": { + "type": "timestamp[ns]", + "comment": "Date (UTC) that the fix was processed at SMRU" + }, + "nsats_healthy": { + "type": "float32", + "comment": "The number of SV 2019s for which valid ephemeris data was available." + }, + "nbits": { + "type": "int32", + "comment": "Resolution of the transmitted pseudoranges" + }, + "deleted": { + "type": "string", + "comment": "The value Y here causes this position to be ignored by the location filtering algorithm." + }, + "v_mask": { + "type": "int32", + "comment": "Result of the location filtering algorithm: 0 = accepted, 20 = rejected at the 20 dm/s level (see VMASK_THRESHOLD). The first and last two points of the track are excluded from the algorithm and are assigned -1" + }, + "km_from_home": { + "type": "double", + "long_name": "Distance from home", + "units": "km", + "comment": "Great-circle distance to the release_latitude, release_longitude position specified in the metadata table (km). This is simply the shortest distance between the two points: the path may cross land." + }, + "est_speed": { + "type": "string", + "long_name": "Estimated speed", + "units": "m/s", + "comment": "Speed across the surface estimated by the location filtering algorithm (m/s)" + }, + "gps_number": { + "type": "string" + }, + "tx_date": { + "type": "string" + }, + "uplink_number": { + "type": "string" + }, + "tagging_id": { + "type": "string" + }, + "d_date_tag": { + "type": "string" + }, + "ssm_lon": { + "type": "float32" + }, + "ssm_lat": { + "type": "float32" + }, + "ssm_x": { + "type": "float32" + }, + "ssm_y": { + "type": "float32" + }, + "ssm_x_se": { + "type": "float32" + }, + "ssm_y_se": { + "type": "float32" + }, + "cid": { + "type": "string", + "comment": "SMRU campaign name " + } + }, + "aws_opendata_registry": { + "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control", + "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5", + "Contact": "info@aodn.org.au", + "ManagedBy": "AODN", + "UpdateFrequency": "As Needed", + "Tags": [ + "FILL UP MANUALLY - CHECK DOCUMENTATION" + ], + "License": "http://creativecommons.org/licenses/by/4.0/", + "Resources": [ + { + "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_gps_satellite_relay_tagging_location_qc_delayed_qc.parquet", + "Region": "ap-southeast-2", + "Type": "S3 Bucket" + } + ], + "DataAtWork": { + "Tutorials": [ + { + "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_gps_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_gps_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + }, + { + "Title": "Accessing and search for any AODN dataset", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + } + ] + }, + "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" + }, + "csv_config": { + "pandas_read_csv_config": { + "na_values": [ + "N/A", + "NaN" + ], + "encoding": "utf-8", + "index_col": "d_date", + "parse_dates": [ + "d_date" + ] + } + }, + "schema_transformation": { + "drop_variables": [], + "add_variables": { + "filename": { + "source": "@filename", + "schema": { + "type": "string", + "units": "1", + "long_name": "Filename of the source file" + } + }, + "timestamp": { + "source": "@partitioning:time_extent", + "schema": { + "type": "int64", + "units": "1", + "long_name": "Partition timestamp" + } + }, + "polygon": { + "source": "@partitioning:spatial_extent", + "schema": { + "type": "string", + "units": "1", + "long_name": "Spatial partition polygon" + } + } + }, + "partitioning": [ + { + "source_variable": "timestamp", + "type": "time_extent", + "time_extent": { + "time_varname": "d_date", + "partition_period": "Y" + } + }, + { + "source_variable": "polygon", + "type": "spatial_extent", + "spatial_extent": { + "lat_varname": "lat", + "lon_varname": "lon", + "spatial_resolution": 20 + } + } + ], + "global_attributes": { + "delete": [ + "geospatial_lat_max", + "geospatial_lat_min", + "geospatial_lon_max", + "geospatial_lon_min", + "date_created" + ], + "set": {} + } + }, + "handler_class": "AatamsSatelliteTaggingHandler" +} diff --git a/aodn_cloud_optimised/config/dataset/animal_haulout_satellite_relay_tagging_location_qc_delayed_qc.json b/aodn_cloud_optimised/config/dataset/animal_haulout_satellite_relay_tagging_location_qc_delayed_qc.json new file mode 100644 index 00000000..ed685ce6 --- /dev/null +++ b/aodn_cloud_optimised/config/dataset/animal_haulout_satellite_relay_tagging_location_qc_delayed_qc.json @@ -0,0 +1,240 @@ +{ + "dataset_name": "animal_haulout_satellite_relay_tagging_location_qc_delayed_qc", + "logger_name": "animal_haulout_satellite_relay_tagging_location_qc_delayed_qc", + "cloud_optimised_format": "parquet", + "run_settings": { + "paths": [ + { + "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM", + "filter": [ + ".*_haulout_.*_dm\\.csv$" + ], + "year_range": [] + } + ], + "cluster": { + "mode": "local", + "restart_every_path": false + }, + "clear_existing_data": true, + "raise_error": false, + "coiled_cluster_options": { + "n_workers": [ + 1, + 20 + ], + "scheduler_vm_types": "m7i-flex.large", + "worker_vm_types": "m7i-flex.large", + "allow_ingress_from": "me", + "compute_purchase_option": "spot_with_fallback", + "worker_options": { + "nthreads": 4, + "memory_limit": "8GB" + } + }, + "batch_size": 5, + "force_previous_parquet_deletion": false + }, + "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5", + "schema": { + "ref": { + "type": "string", + "long_name": "reference", + "comment": "A three-part code (e.g. rs3-Fred-07) : The first part identifies the batch of tags, the second part is the name or number of the animal, the third part is the year" + }, + "ptt": { + "type": "int32", + "long_name": "Platform Transmitter Terminal", + "comment": "Argos PTT number is used" + }, + "s_date": { + "type": "timestamp[ns]", + "long_name": "Start date", + "comment": "the time (UTC) that the haulout started. It takes 10 minutes of uninterrupted dry readings to trigger a haulout these 10 minutes are included." + }, + "e_date": { + "type": "timestamp[ns]", + "long_name": "End date", + "comment": "The time (UTC) that the haulout ended" + }, + "haulout_number": { + "type": "int32", + "comment": "The SRDL assigns an incrementing number to each haulout that occurs. The value stored in this field returns to 0 when it reaches the limit of the resolution used to transmit it (normally 0..31 or 0..63). When two haulouts with consecutive numbers are received we can infer that no other haulout occurred between the end of the first haulout and the beginning of the second one. These periods of known non-haulout are shown as narrow grey bars connecting the green haulout bars on the MamVis strip chart display." + }, + "cnt": { + "type": "int32", + "comment": "The number of times that this record was received" + }, + "phosi_secs": { + "type": "int32" + }, + "wet_n": { + "type": "double" + }, + "wet_min": { + "type": "double" + }, + "wet_max": { + "type": "double" + }, + "wet_mean": { + "type": "double" + }, + "wet_sd": { + "type": "double" + }, + "tagging_id": { + "type": "double" + }, + "s_date_tag": { + "type": "string" + }, + "e_date_tag": { + "type": "string" + }, + "end_number": { + "type": "int32", + "comment": "In this table HAULOUT_NUMBER to END_NUMBER give the range of transmitted records that were concatenated to form this record." + }, + "lat": { + "type": "double" + }, + "lon": { + "type": "double" + }, + "ssm_lon": { + "type": "double" + }, + "ssm_lat": { + "type": "double" + }, + "ssm_x": { + "type": "double" + }, + "ssm_y": { + "type": "double" + }, + "ssm_x_se": { + "type": "double" + }, + "ssm_y_se": { + "type": "double" + }, + "cid": { + "type": "string", + "comment": "SMRU campaign name " + } + }, + "aws_opendata_registry": { + "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control", + "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5", + "Contact": "info@aodn.org.au", + "ManagedBy": "AODN", + "UpdateFrequency": "As Needed", + "Tags": [ + "FILL UP MANUALLY - CHECK DOCUMENTATION" + ], + "License": "http://creativecommons.org/licenses/by/4.0/", + "Resources": [ + { + "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_haulout_satellite_relay_tagging_location_qc_delayed_qc.parquet", + "Region": "ap-southeast-2", + "Type": "S3 Bucket" + } + ], + "DataAtWork": { + "Tutorials": [ + { + "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_haulout_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_haulout_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + }, + { + "Title": "Accessing and search for any AODN dataset", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + } + ] + }, + "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" + }, + "csv_config": { + "pandas_read_csv_config": { + "na_values": [ + "N/A", + "NaN" + ], + "encoding": "utf-8", + "index_col": "s_date", + "parse_dates": [ + "s_date", + "e_date" + ] + } + }, + "schema_transformation": { + "drop_variables": [], + "add_variables": { + "filename": { + "source": "@filename", + "schema": { + "type": "string", + "units": "1", + "long_name": "Filename of the source file" + } + }, + "timestamp": { + "source": "@partitioning:time_extent", + "schema": { + "type": "int64", + "units": "1", + "long_name": "Partition timestamp" + } + }, + "polygon": { + "source": "@partitioning:spatial_extent", + "schema": { + "type": "string", + "units": "1", + "long_name": "Spatial partition polygon" + } + } + }, + "partitioning": [ + { + "source_variable": "timestamp", + "type": "time_extent", + "time_extent": { + "time_varname": "s_date", + "partition_period": "Y" + } + }, + { + "source_variable": "polygon", + "type": "spatial_extent", + "spatial_extent": { + "lat_varname": "lat", + "lon_varname": "lon", + "spatial_resolution": 20 + } + } + ], + "global_attributes": { + "delete": [ + "geospatial_lat_max", + "geospatial_lat_min", + "geospatial_lon_max", + "geospatial_lon_min", + "date_created" + ], + "set": {} + } + }, + "handler_class": "AatamsSatelliteTaggingHandler" +} diff --git a/aodn_cloud_optimised/config/dataset/animal_metadata_satellite_relay_tagging_location_qc_delayed_qc.json b/aodn_cloud_optimised/config/dataset/animal_metadata_satellite_relay_tagging_location_qc_delayed_qc.json new file mode 100644 index 00000000..dad6b40b --- /dev/null +++ b/aodn_cloud_optimised/config/dataset/animal_metadata_satellite_relay_tagging_location_qc_delayed_qc.json @@ -0,0 +1,255 @@ +{ + "dataset_name": "animal_metadata_satellite_relay_tagging_location_qc_delayed_qc", + "logger_name": "animal_metadata_satellite_relay_tagging_location_qc_delayed_qc", + "cloud_optimised_format": "parquet", + "run_settings": { + "paths": [ + { + "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM", + "filter": [ + ".*_metadata_.*_dm\\.csv$" + ], + "year_range": [] + } + ], + "cluster": { + "mode": "local", + "restart_every_path": false + }, + "clear_existing_data": true, + "raise_error": false, + "coiled_cluster_options": { + "n_workers": [ + 1, + 20 + ], + "scheduler_vm_types": "m7i-flex.large", + "worker_vm_types": "m7i-flex.large", + "allow_ingress_from": "me", + "compute_purchase_option": "spot_with_fallback", + "worker_options": { + "nthreads": 4, + "memory_limit": "8GB" + } + }, + "batch_size": 5, + "force_previous_parquet_deletion": false + }, + "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5", + "schema": { + "sattag_program": { + "type": "string", + "comment": "SMRU campaign name " + }, + "device_id": { + "type": "string", + "comment": "SMRU reference ID " + }, + "ptt": { + "type": "int32", + "comment": "Argos platform terminal transmitter ID" + }, + "body": { + "type": "int32", + "comment": "SMRU tag ID " + }, + "device_wmo_ref": { + "type": "string", + "comment": "WMO ID " + }, + "tag_type": { + "type": "string", + "comment": "SMRU tag type designation" + }, + "common_name": { + "type": "string", + "comment": "Deployment animal common name" + }, + "species": { + "type": "string", + "comment": "Deployment animal scientific name" + }, + "release_longitude": { + "type": "double", + "units": "degrees_east", + "comment": "Tag deployment coordinate (WGS 84 datum)" + }, + "release_latitude": { + "type": "double", + "units": "degrees_north", + "comment": "Tag deployment coordinate (WGS 84 datum)" + }, + "release_site": { + "type": "string", + "comment": "Tag deployment locality" + }, + "release_date": { + "type": "timestamp[ns]", + "comment": "Tag deployment date (UTC)" + }, + "recovery_date": { + "type": "string", + "comment": "Tag recovery date (UTC)" + }, + "age_class": { + "type": "string", + "comment": "Animal age class (e.g. Juvenile; adult; NA)" + }, + "sex": { + "type": "string", + "comment": "Animal sex (e.g. m; f; NA)" + }, + "length": { + "type": "double", + "units": "m", + "comment": "Animal length (m)" + }, + "estimated_mass": { + "type": "double", + "units": "kg", + "comment": "Animal mass (kg) researcher estimated" + }, + "actual_mass": { + "type": "double", + "units": "kg", + "comment": "Animal mass (kg) measurement" + }, + "state_country": { + "type": "string", + "comment": "Country of deployment " + }, + "qc_start_date": { + "type": "timestamp[ns]", + "comment": "Time of first data record included in QC (UTC)" + }, + "qc_end_date": { + "type": "timestamp[ns]", + "comment": "Time of last data record included in QC (UTC)" + }, + "qc_method": { + "type": "string" + }, + "qc_method_version": { + "type": "string" + }, + "qc_proj4string": { + "type": "string" + }, + "qc_run_date": { + "type": "string" + } + }, + "aws_opendata_registry": { + "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control", + "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5", + "Contact": "info@aodn.org.au", + "ManagedBy": "AODN", + "UpdateFrequency": "As Needed", + "Tags": [ + "FILL UP MANUALLY - CHECK DOCUMENTATION" + ], + "License": "http://creativecommons.org/licenses/by/4.0/", + "Resources": [ + { + "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_metadata_satellite_relay_tagging_location_qc_delayed_qc.parquet", + "Region": "ap-southeast-2", + "Type": "S3 Bucket" + } + ], + "DataAtWork": { + "Tutorials": [ + { + "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_metadata_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_metadata_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + }, + { + "Title": "Accessing and search for any AODN dataset", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + } + ] + }, + "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" + }, + "csv_config": { + "pandas_read_csv_config": { + "na_values": [ + "N/A", + "NaN" + ], + "encoding": "utf-8", + "index_col": "qc_start_date", + "parse_dates": [ + "release_date", + "qc_start_date", + "qc_end_date" + ] + } + }, + "schema_transformation": { + "drop_variables": [], + "add_variables": { + "filename": { + "source": "@filename", + "schema": { + "type": "string", + "units": "1", + "long_name": "Filename of the source file" + } + }, + "timestamp": { + "source": "@partitioning:time_extent", + "schema": { + "type": "int64", + "units": "1", + "long_name": "Partition timestamp" + } + }, + "polygon": { + "source": "@partitioning:spatial_extent", + "schema": { + "type": "string", + "units": "1", + "long_name": "Spatial partition polygon" + } + } + }, + "partitioning": [ + { + "source_variable": "timestamp", + "type": "time_extent", + "time_extent": { + "time_varname": "release_date", + "partition_period": "M" + } + }, + { + "source_variable": "polygon", + "type": "spatial_extent", + "spatial_extent": { + "lat_varname": "release_latitude", + "lon_varname": "release_longitude", + "spatial_resolution": 5 + } + } + ], + "global_attributes": { + "delete": [ + "geospatial_lat_max", + "geospatial_lat_min", + "geospatial_lon_max", + "geospatial_lon_min", + "date_created" + ], + "set": {} + } + }, + "handler_class": "AatamsSatelliteTaggingHandler" +} diff --git a/aodn_cloud_optimised/config/dataset/animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc.json b/aodn_cloud_optimised/config/dataset/animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc.json new file mode 100644 index 00000000..fa23ef6e --- /dev/null +++ b/aodn_cloud_optimised/config/dataset/animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc.json @@ -0,0 +1,206 @@ +{ + "dataset_name": "animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc", + "logger_name": "animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc", + "cloud_optimised_format": "parquet", + "run_settings": { + "paths": [ + { + "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM", + "filter": [ + ".*_ssmoutputs_.*_dm\\.csv$" + ], + "year_range": [] + } + ], + "cluster": { + "mode": "local", + "restart_every_path": false + }, + "clear_existing_data": true, + "raise_error": false, + "coiled_cluster_options": { + "n_workers": [ + 1, + 20 + ], + "scheduler_vm_types": "m7i-flex.large", + "worker_vm_types": "m7i-flex.large", + "allow_ingress_from": "me", + "compute_purchase_option": "spot_with_fallback", + "worker_options": { + "nthreads": 4, + "memory_limit": "8GB" + } + }, + "batch_size": 5, + "force_previous_parquet_deletion": false + }, + "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5", + "schema": { + "ref": { + "type": "string", + "long_name": "reference", + "comment": "A three-part code (e.g. rs3-Fred-07) : The first part identifies the batch of tags, the second part is the name or number of the animal, the third part is the year" + }, + "date": { + "type": "timestamp[ns]" + }, + "lon": { + "type": "double", + "long_name": "Longitude", + "units": "degrees_east", + "comment": "Position estimate (WGS 84 datum)" + }, + "lat": { + "type": "double", + "long_name": "Latitude", + "units": "degrees_north", + "comment": "Position estimate (WGS 84 datum)" + }, + "x": { + "type": "double" + }, + "y": { + "type": "double" + }, + "x_se": { + "type": "double" + }, + "y_se": { + "type": "double" + }, + "u": { + "type": "double" + }, + "v": { + "type": "double" + }, + "u_se": { + "type": "double" + }, + "v_se": { + "type": "double" + }, + "s": { + "type": "double" + }, + "s_se": { + "type": "double" + }, + "cid": { + "type": "string", + "comment": "SMRU campaign name " + } + }, + "aws_opendata_registry": { + "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control", + "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5", + "Contact": "info@aodn.org.au", + "ManagedBy": "AODN", + "UpdateFrequency": "As Needed", + "Tags": [ + "FILL UP MANUALLY - CHECK DOCUMENTATION" + ], + "License": "http://creativecommons.org/licenses/by/4.0/", + "Resources": [ + { + "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc.parquet", + "Region": "ap-southeast-2", + "Type": "S3 Bucket" + } + ], + "DataAtWork": { + "Tutorials": [ + { + "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_ssmoutputs_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + }, + { + "Title": "Accessing and search for any AODN dataset", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + } + ] + }, + "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" + }, + "csv_config": { + "pandas_read_csv_config": { + "na_values": [ + "N/A", + "NaN" + ], + "encoding": "utf-8", + "index_col": "date", + "parse_dates": [ + "date" + ] + } + }, + "schema_transformation": { + "drop_variables": [], + "add_variables": { + "filename": { + "source": "@filename", + "schema": { + "type": "string", + "units": "1", + "long_name": "Filename of the source file" + } + }, + "timestamp": { + "source": "@partitioning:time_extent", + "schema": { + "type": "int64", + "units": "1", + "long_name": "Partition timestamp" + } + }, + "polygon": { + "source": "@partitioning:spatial_extent", + "schema": { + "type": "string", + "units": "1", + "long_name": "Spatial partition polygon" + } + } + }, + "partitioning": [ + { + "source_variable": "timestamp", + "type": "time_extent", + "time_extent": { + "time_varname": "date", + "partition_period": "Y" + } + }, + { + "source_variable": "polygon", + "type": "spatial_extent", + "spatial_extent": { + "lat_varname": "lat", + "lon_varname": "lon", + "spatial_resolution": 20 + } + } + ], + "global_attributes": { + "delete": [ + "geospatial_lat_max", + "geospatial_lat_min", + "geospatial_lon_max", + "geospatial_lon_min", + "date_created" + ], + "set": {} + } + }, + "handler_class": "AatamsSatelliteTaggingHandler" +} diff --git a/aodn_cloud_optimised/config/dataset/animal_summary_satellite_relay_tagging_location_qc_delayed_qc.json b/aodn_cloud_optimised/config/dataset/animal_summary_satellite_relay_tagging_location_qc_delayed_qc.json new file mode 100644 index 00000000..64ba2ae8 --- /dev/null +++ b/aodn_cloud_optimised/config/dataset/animal_summary_satellite_relay_tagging_location_qc_delayed_qc.json @@ -0,0 +1,367 @@ +{ + "dataset_name": "animal_summary_satellite_relay_tagging_location_qc_delayed_qc", + "logger_name": "animal_summary_satellite_relay_tagging_location_qc_delayed_qc", + "cloud_optimised_format": "parquet", + "run_settings": { + "paths": [ + { + "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM", + "filter": [ + ".*_summary_.*_dm\\.csv$" + ], + "year_range": [] + } + ], + "cluster": { + "mode": "local", + "restart_every_path": false + }, + "clear_existing_data": true, + "raise_error": false, + "coiled_cluster_options": { + "n_workers": [ + 1, + 20 + ], + "scheduler_vm_types": "m7i-flex.large", + "worker_vm_types": "m7i-flex.large", + "allow_ingress_from": "me", + "compute_purchase_option": "spot_with_fallback", + "worker_options": { + "nthreads": 4, + "memory_limit": "8GB" + } + }, + "batch_size": 5, + "force_previous_parquet_deletion": false + }, + "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5", + "schema": { + "ref": { + "type": "string", + "long_name": "reference", + "comment": "A three-part code (e.g. rs3-Fred-07) : The first part identifies the batch of tags, the second part is the name or number of the animal, the third part is the year" + }, + "ptt": { + "type": "int32", + "long_name": "Platform Transmitter Terminal", + "comment": "Argos PTT number is used" + }, + "cnt": { + "type": "int32", + "comment": "The number of times that this record was received" + }, + "s_date": { + "type": "timestamp[ns]", + "comment": "The start time (UTC) of the summary period" + }, + "e_date": { + "type": "timestamp[ns]", + "comment": "The end time (UTC) of the summary period. Note that dives are assigned to the period in which they ended for the purposes of calculating the summary statistics described below" + }, + "div_dist": { + "type": "double" + }, + "surf_tm": { + "type": "double", + "comment": "The proportion of time spent at the surface (%). This includes the SURF_DUR periods described above, plus any other periods which were neither dives nor haulouts" + }, + "dive_tm": { + "type": "double", + "comment": "The proportion of time spent diving (%). This means the time between the detection of the start of the dive and the detection of the end of the dive. The subsequent SURF_DUR time at the surface that is included in the individual dive records does not contribute to DIVE_TM." + }, + "haul_tm": { + "type": "double", + "comment": "The proportion of time spent hauled-out (%)" + }, + "n_cycles": { + "type": "int32", + "comment": "The number of dives that finished during the summary period" + }, + "av_depth": { + "type": "double", + "units": "m", + "comment": "The average max_depth in the summary period (metres), where max_depth is the maximum depth recorded in a given dive." + }, + "max_depth": { + "type": "double", + "units": "m", + "comment": "The maximum dive depth (metres)" + }, + "cruise_tm": { + "type": "double" + }, + "avg_sst": { + "type": "double" + }, + "avg_speed": { + "type": "double", + "units": "m/s", + "comment": "The average speed recorded during dives (metres/second)" + }, + "sd_depth": { + "type": "double", + "units": "m", + "comment": "The standard deviation of max_depth (metres)" + }, + "av_dur": { + "type": "int32", + "units": "s", + "comment": "The average dive duration (seconds)" + }, + "sd_dur": { + "type": "int32", + "units": "s", + "comment": "The standard deviation of dive duration (seconds)" + }, + "max_dur": { + "type": "int32", + "units": "s", + "comment": "The maximum dive duration (seconds)" + }, + "dp_n_cycles": { + "type": "double" + }, + "dp_av_depth": { + "type": "double", + "comment": "These fields will be present where the SRDL has been programmed to separate deep from shallow dives according to a maximum depth threshold. These fields hold statistics calculated for the deep dives, the corresponding fields above hold the values for shallow dives." + }, + "dp_max_depth": { + "type": "double", + "comment": "These fields will be present where the SRDL has been programmed to separate deep from shallow dives according to a maximum depth threshold. These fields hold statistics calculated for the deep dives, the corresponding fields above hold the values for shallow dives." + }, + "dp_avg_speed": { + "type": "double", + "comment": "These fields will be present where the SRDL has been programmed to separate deep from shallow dives according to a maximum depth threshold. These fields hold statistics calculated for the deep dives, the corresponding fields above hold the values for shallow dives." + }, + "dp_sd_depth": { + "type": "double", + "comment": "These fields will be present where the SRDL has been programmed to separate deep from shallow dives according to a maximum depth threshold. These fields hold statistics calculated for the deep dives, the corresponding fields above hold the values for shallow dives." + }, + "dp_av_dur": { + "type": "double", + "comment": "These fields will be present where the SRDL has been programmed to separate deep from shallow dives according to a maximum depth threshold. These fields hold statistics calculated for the deep dives, the corresponding fields above hold the values for shallow dives." + }, + "dp_sd_dur": { + "type": "double", + "comment": "These fields will be present where the SRDL has been programmed to separate deep from shallow dives according to a maximum depth threshold. These fields hold statistics calculated for the deep dives, the corresponding fields above hold the values for shallow dives." + }, + "dp_max_dur": { + "type": "double", + "comment": "These fields will be present where the SRDL has been programmed to separate deep from shallow dives according to a maximum depth threshold. These fields hold statistics calculated for the deep dives, the corresponding fields above hold the values for shallow dives." + }, + "dp_dive_tm": { + "type": "double", + "comment": "The proportion of DIVE_TM that was spent in deep dives (%). i.e. the percentage of time in deep dives = DIVE_TM x DP_DIVE_TM/100; the proportion of time in shallow dives = DIVE_TM x (1 - DP_DIVE_TM/100)" + }, + "av_surf_dur": { + "type": "double", + "units": "s", + "comment": "The average post-dive surface duration (seconds)" + }, + "sd_surf_dur": { + "type": "double", + "units": "s", + "comment": "The standard deviation of post-dive surface duration (seconds)" + }, + "max_surf_dur": { + "type": "double", + "units": "s", + "comment": "The maximum post-dive surface duration (seconds)" + }, + "dp_av_surf_dur": { + "type": "double" + }, + "dp_sd_surf_dur": { + "type": "double" + }, + "dp_max_surf_dur": { + "type": "double" + }, + "pca": { + "type": "double" + }, + "swim_eff_desc": { + "type": "double" + }, + "swim_eff_asc": { + "type": "double" + }, + "swim_eff_whole": { + "type": "double" + }, + "secs_desc": { + "type": "double" + }, + "secs_asc": { + "type": "double" + }, + "pitch_desc": { + "type": "double" + }, + "pitch_asc": { + "type": "double" + }, + "av_haulout_dur": { + "type": "double" + }, + "sd_haulout_dur": { + "type": "double" + }, + "max_haulout_dur": { + "type": "double" + }, + "av_phosi_dur": { + "type": "double" + }, + "sd_phosi_dur": { + "type": "double" + }, + "max_phosi_dur": { + "type": "double" + }, + "tagging_id": { + "type": "double" + }, + "s_date_tag": { + "type": "string" + }, + "e_date_tag": { + "type": "string" + }, + "ssm_lon": { + "type": "double" + }, + "ssm_lat": { + "type": "double" + }, + "ssm_x": { + "type": "double" + }, + "ssm_y": { + "type": "double" + }, + "ssm_x_se": { + "type": "double" + }, + "ssm_y_se": { + "type": "double" + }, + "cid": { + "type": "string", + "comment": "SMRU campaign name " + } + }, + "aws_opendata_registry": { + "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control", + "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5", + "Contact": "info@aodn.org.au", + "ManagedBy": "AODN", + "UpdateFrequency": "As Needed", + "Tags": [ + "FILL UP MANUALLY - CHECK DOCUMENTATION" + ], + "License": "http://creativecommons.org/licenses/by/4.0/", + "Resources": [ + { + "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_summary_satellite_relay_tagging_location_qc_delayed_qc.parquet", + "Region": "ap-southeast-2", + "Type": "S3 Bucket" + } + ], + "DataAtWork": { + "Tutorials": [ + { + "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_summary_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_summary_satellite_relay_tagging_location_qc_delayed_qc.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + }, + { + "Title": "Accessing and search for any AODN dataset", + "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb", + "AuthorName": "Laurent Besnard", + "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised" + } + ] + }, + "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" + }, + "csv_config": { + "pandas_read_csv_config": { + "na_values": [ + "N/A", + "NaN" + ], + "encoding": "utf-8", + "index_col": "s_date", + "parse_dates": [ + "s_date", + "e_date" + ] + } + }, + "schema_transformation": { + "drop_variables": [], + "add_variables": { + "filename": { + "source": "@filename", + "schema": { + "type": "string", + "units": "1", + "long_name": "Filename of the source file" + } + }, + "timestamp": { + "source": "@partitioning:time_extent", + "schema": { + "type": "int64", + "units": "1", + "long_name": "Partition timestamp" + } + }, + "polygon": { + "source": "@partitioning:spatial_extent", + "schema": { + "type": "string", + "units": "1", + "long_name": "Spatial partition polygon" + } + } + }, + "partitioning": [ + { + "source_variable": "timestamp", + "type": "time_extent", + "time_extent": { + "time_varname": "s_date", + "partition_period": "Y" + } + }, + { + "source_variable": "polygon", + "type": "spatial_extent", + "spatial_extent": { + "lat_varname": "ssm_lat", + "lon_varname": "ssm_lon", + "spatial_resolution": 20 + } + } + ], + "global_attributes": { + "delete": [ + "geospatial_lat_max", + "geospatial_lat_min", + "geospatial_lon_max", + "geospatial_lon_min", + "date_created" + ], + "set": {} + } + }, + "handler_class": "AatamsSatelliteTaggingHandler" +}