add sattag dm config only

aodn_cloud_optimised/config/dataset/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.json

@@ -0,0 +1,307 @@
+{
+  "dataset_name": "animal_ctd_satellite_relay_tagging_location_qc_delayed_qc",
+  "logger_name": "animal_ctd_satellite_relay_tagging_location_qc_delayed_qc",
+  "cloud_optimised_format": "parquet",
+  "run_settings": {
+    "paths": [
+      {
+        "s3_uri": "s3://imos-data/IMOS/AATAMS/satellite_tagging/ATF_Location_QC_DM",
+        "filter": [
+          ".*_ctd_.*_dm\\.csv$"
+        ],
+        "year_range": []
+      }
+    ],
+    "cluster": {
+      "mode": "local",
+      "restart_every_path": false
+    },
+    "clear_existing_data": true,
+    "raise_error": false,
+    "coiled_cluster_options": {
+      "n_workers": [
+        1,
+        20
+      ],
+      "scheduler_vm_types": "m7i-flex.large",
+      "worker_vm_types": "m7i-flex.large",
+      "allow_ingress_from": "me",
+      "compute_purchase_option": "spot_with_fallback",
+      "worker_options": {
+        "nthreads": 4,
+        "memory_limit": "8GB"
+      }
+    },
+    "batch_size": 5,
+    "force_previous_parquet_deletion": false
+  },
+  "metadata_uuid": "b2548767-514f-4a31-b65e-36bb894382d5",
+  "schema": {
+    "ref": {
+      "type": "string",
+      "long_name": "reference",
+      "comment": "A three-part code (e.g. rs3-Fred-07) : The first part identifies the batch of tags, the second part is the name or number of the animal, the third part is the year"
+    },
+    "ptt": {
+      "type": "int32",
+      "long_name": "Platform Transmitter Terminal",
+      "comment": "Argos PTT number is used"
+    },
+    "end_date": {
+      "type": "timestamp[ns]",
+      "comment": "The time (UTC) of the end of the upcast"
+    },
+    "max_dbar": {
+      "type": "double",
+      "long_name": "Maximum pressure",
+      "comment": "The maximum pressure (in decibar, approximately 1 metre)"
+    },
+    "num": {
+      "type": "int32",
+      "long_name": "Upcast number",
+      "comment": "Upcasts are assigned sequential numbers as they are performed. The SRDL transmits only the deepest casts that occur in each period (usually 6 hours or 12 hours). This means that some earlier upcasts will not be transmitted so these numbers will not be consecutive."
+    },
+    "n_temp": {
+      "type": "int32",
+      "comment": "The number of (pressure,temperature) pairs "
+    },
+    "n_cond": {
+      "type": "double",
+      "comment": "The number of (pressure,conductivity) pairs "
+    },
+    "n_sal": {
+      "type": "int32",
+      "comment": "The number of (pressure,salinity) pairs "
+    },
+    "temp_dbar": {
+      "type": "double",
+      "long_name": "Pressure coordinates of the (pressure,temperature) profile",
+      "units": "decibars",
+      "comment": "A comma-separated list of the pressure coordinates of the (pressure,temperature) profile (decibars)"
+    },
+    "temp_vals": {
+      "type": "double",
+      "long_name": "Temperature profile",
+      "units": "degrees Celsius",
+      "comment": "A comma-separated list of the temperature coordinates of the (pressure,temperature) profile (degree C)"
+    },
+    "cond_dbar": {
+      "type": "double",
+      "long_name": "Pressure coordinates of the (pressure,conductivity) profile",
+      "units": "decibars",
+      "comment": "A comma-separated list of the pressure coordinates of the (pressure,conductivity) profile (decibars)"
+    },
+    "cond_vals": {
+      "type": "double",
+      "long_name": "Conductivity profile",
+      "units": "mS/cm",
+      "comment": "A comma-separated list of the conductivity coordinates of the (pressure,conductivity) profile (mS/cm)"
+    },
+    "sal_dbar": {
+      "type": "double",
+      "long_name": "Pressure coordinates of the (pressure,salinity) profile",
+      "units": "decibars",
+      "comment": "A comma-separated list of the pressure coordinates of the (pressure,salinity) profile (decibars)"
+    },
+    "sal_vals": {
+      "type": "double",
+      "long_name": "Salinity profile",
+      "units": " ",
+      "comment": "A comma-separated list of the salinity coordinates of the (pressure,salinity) profile"
+    },
+    "n_fluoro": {
+      "type": "double"
+    },
+    "fluoro_dbar": {
+      "type": "double",
+      "units": "decibars"
+    },
+    "fluoro_vals": {
+      "type": "double"
+    },
+    "n_oxy": {
+      "type": "double"
+    },
+    "oxy_dbar": {
+      "type": "double",
+      "units": "decibars"
+    },
+    "oxy_vals": {
+      "type": "double"
+    },
+    "qc_profile": {
+      "type": "int32"
+    },
+    "qc_temp": {
+      "type": "double"
+    },
+    "qc_sal": {
+      "type": "double"
+    },
+    "sal_corrected_vals": {
+      "type": "double"
+    },
+    "created": {
+      "type": "string",
+      "long_name": "Date of file creation (UTC)"
+    },
+    "modified": {
+      "type": "string",
+      "long_name": "Date of update of this file (UTC)"
+    },
+    "n_photo": {
+      "type": "double"
+    },
+    "photo_dbar": {
+      "type": "double"
+    },
+    "photo_vals": {
+      "type": "double"
+    },
+    "lat": {
+      "type": "double",
+      "long_name": "Latitude",
+      "units": "degrees_north",
+      "comment": "Position estimate (WGS 84 datum)"
+    },
+    "lon": {
+      "type": "double",
+      "long_name": "Longitude",
+      "units": "degrees_east",
+      "comment": "Position estimate (WGS 84 datum)"
+    },
+    "ssm_lon": {
+      "type": "double"
+    },
+    "ssm_lat": {
+      "type": "double"
+    },
+    "ssm_x": {
+      "type": "double"
+    },
+    "ssm_y": {
+      "type": "double"
+    },
+    "ssm_x_se": {
+      "type": "double"
+    },
+    "ssm_y_se": {
+      "type": "double"
+    },
+    "cid": {
+      "type": "string",
+      "comment": "SMRU campaign name "
+    }
+  },
+  "aws_opendata_registry": {
+    "Name": "IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations",
+    "Description": "The Animal Tagging Sub-Facility of the IMOS Animal Tracking Facility (formerly known as the Australian Animal Tracking And Monitoring System (AATAMS)) is a coordinated marine animal satellite-tracking project. Satellite Relay Data Loggers (SRDL) (most with Conductivity-Temperature-Depth (CTD) sensors, and some with CTD/fluorometers) are used to explore how marine animals (e.g. mammals) interact with their oceanic environment. The primary focus of this program is to collect CTD data from marine mammals, such as southern elephant seals (Mirounga leonina) and Weddell seals (Leptonychotes weddellii), in the southern Indian Ocean and along the East Antarctic coast. This is achieved through international collaborations with the French national polar program's long-term National Observatory Services: Mammals as Ocean Bio-Samplers (SNO-MEMO), the National Institute of Water and Atmospheric Research as part of the Ross Sea Research and Monitoring Programme, and the CNES (Centre National d'Etudes Spatiales) TOSCA programme (Phoques de Weddell bio-oceanographes de la banquise antarctique et outils satellites) in Terre Adelie. SRDL loggers developed at the Sea Mammal Research Unit (SMRU, University of St Andrews, UK) transmit data in delayed mode via the Argos satellite system. This metadata record, represents several different datasets listed hereafter, which can all be accessed through a multi-WFS service. \nThe data represented by this record are presented in delayed mode. \nCTD - parameters measured by the instruments include time, depth, conductivity (salinity), temperature, speed and fluorescence (available in some deployments). \nDiving - parameters measured by the instruments include individual dive start and end time, longitude/latitude at dive end time, post-dive surface duration, dive duration, maximum dive depth, intermediate dive depths and times. \nHaulout - a haulout begins when the SRDL has been continuously dry for a specified length of time (usually 10 minutes). It ends when continuously wet for another interval (usually 40 seconds). Haulout data parameters measured by the instruments include haulout start and end dates and longitude/latitude, and haulout number. \nArgos Locations - location data parameters measured by the instruments include time, longitude, latitude, location quality, along with other diagnostic information provided by Argos (http://www.argos-system.org/). \nSummary Statistics - as well as sending records of individual events such as dives and haulouts, the SRDL also calculates summary statistics of those events over a specified time period (usually 3, 4 or 6 hours). Summary statistics computed by the instruments include the proportion of time spent diving, at the surface and hauled-out, the number of dives, and the average, standard deviation and maximum dive duration and dive depth during each summary period. These statistics are based on all the data recorded by the SRDL and thus are not prone to distortion by variations in the efficiency of transmission via Argos. \nSSM QC Locations - SSM-predicted locations and uncertainty at 6-h intervals from the location quality-control process. SSM-predicted locations are also appended to records in the CTD, Diving, Haulout, Argos Locations, and Summary Statistics datasets. See Jonsen et al. 2020 for further details on the SSM used for location quality control",
+    "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/b2548767-514f-4a31-b65e-36bb894382d5",
+    "Contact": "info@aodn.org.au",
+    "ManagedBy": "AODN",
+    "UpdateFrequency": "As Needed",
+    "Tags": [
+      "FILL UP MANUALLY - CHECK DOCUMENTATION"
+    ],
+    "License": "http://creativecommons.org/licenses/by/4.0/",
+    "Resources": [
+      {
+        "Description": "Cloud Optimised AODN dataset of IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations",
+        "ARN": "arn:aws:s3:::aodn-cloud-optimised/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.parquet",
+        "Region": "ap-southeast-2",
+        "Type": "S3 Bucket"
+      }
+    ],
+    "DataAtWork": {
+      "Tutorials": [
+        {
+          "Title": "Accessing IMOS - Animal Tracking Facility - Satellite Relay Tagging Program - Delayed mode data with quality-controlled locations",
+          "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.ipynb",
+          "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_ctd_satellite_relay_tagging_location_qc_delayed_qc.ipynb",
+          "AuthorName": "Laurent Besnard",
+          "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised"
+        },
+        {
+          "Title": "Accessing and search for any AODN dataset",
+          "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb",
+          "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb",
+          "AuthorName": "Laurent Besnard",
+          "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised"
+        }
+      ]
+    },
+    "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]"
+  },
+  "csv_config": {
+    "pandas_read_csv_config": {
+      "na_values": [
+        "N/A",
+        "NaN"
+      ],
+      "encoding": "utf-8",
+      "index_col": "end_date",
+      "parse_dates": [
+        "end_date"
+      ]
+    }
+  },
+  "schema_transformation": {
+    "drop_variables": [],
+    "add_variables": {
+      "filename": {
+        "source": "@filename",
+        "schema": {
+          "type": "string",
+          "units": "1",
+          "long_name": "Filename of the source file"
+        }
+      },
+      "timestamp": {
+        "source": "@partitioning:time_extent",
+        "schema": {
+          "type": "int64",
+          "units": "1",
+          "long_name": "Partition timestamp"
+        }
+      },
+      "polygon": {
+        "source": "@partitioning:spatial_extent",
+        "schema": {
+          "type": "string",
+          "units": "1",
+          "long_name": "Spatial partition polygon"
+        }
+      }
+    },
+    "partitioning": [
+      {
+        "source_variable": "timestamp",
+        "type": "time_extent",
+        "time_extent": {
+          "time_varname": "end_date",
+          "partition_period": "Y"
+        }
+      },
+      {
+        "source_variable": "polygon",
+        "type": "spatial_extent",
+        "spatial_extent": {
+          "lat_varname": "lat",
+          "lon_varname": "lon",
+          "spatial_resolution": 20
+        }
+      }
+    ],
+    "global_attributes": {
+      "delete": [
+        "geospatial_lat_max",
+        "geospatial_lat_min",
+        "geospatial_lon_max",
+        "geospatial_lon_min",
+        "date_created"
+      ],
+      "set": {}
+    }
+  },
+  "handler_class": "AatamsSatelliteTaggingHandler"
+}