Ingest different DRS formats

This PR is a significant rewrite of how we handle DRS file paths to allow us to ingest paths of a few different known DRS formats.

This involved:

• creating a new crate to handle just loading the different DRS types
• changing ingestion to load the different types and then convert them to our metadata format
• fix some funky issues with data-dir not behaving how it should
  • I also made it optional so not having it means that all datasets are ingested

drs/src/parser/cordex.rs

@@ -3,13 +3,13 @@ use std::str::FromStr;
 use chrono::NaiveDateTime;
 use nom::{
     bytes::complete::tag,
-    combinator::{eof, map_res, opt, verify},
+    combinator::{eof, map_res, opt},
     sequence::{preceded, terminated},
     IResult,
 };
 
 use crate::{
-    cordex::{Frequency, PathMetadata, ACTIVITY_VALUE, PRODUCT_VALUE},
+    cordex::{Frequency, PathMetadata},
     parser::{
         cmip5::parse_ensemble,
         common::{name_segment, name_sep, parse_time, parse_version, path_sep},
@@ -24,8 +24,8 @@ pub(crate) fn parse_cordex(i: &str) -> IResult<&str, PathMetadata> {
     <gcm_model_name>/<cmip5_experiement_name>/<cmip5_ensemble_member>/
     <rcm_model_name>/<rcm_version_id>/<frequency>/<variable_name>
     */
-    let (i, _activity) = terminated(parse_activity, path_sep)(i)?;
-    let (i, _product) = terminated(parse_product, path_sep)(i)?;
+    let (i, activity) = path_segment(i)?;
+    let (i, product) = path_segment(i)?;
     let (i, domain) = path_segment(i)?;
     let (i, institution) = path_segment(i)?;
     let (i, gcm_model_name) = path_segment(i)?;
@@ -59,6 +59,8 @@ pub(crate) fn parse_cordex(i: &str) -> IResult<&str, PathMetadata> {
     Ok((
         i,
         PathMetadata {
+            activity,
+            product,
             domain,
             institution,
             gcm_model_name,
@@ -75,14 +77,6 @@ pub(crate) fn parse_cordex(i: &str) -> IResult<&str, PathMetadata> {
     ))
 }
 
-fn parse_activity(i: &str) -> IResult<&str, &str> {
-    verify(word, |activity: &str| activity == ACTIVITY_VALUE)(i)
-}
-
-fn parse_product(i: &str) -> IResult<&str, &str> {
-    verify(word, |product: &str| product == PRODUCT_VALUE)(i)
-}
-
 fn parse_frequency(i: &str) -> IResult<&str, Frequency> {
     map_res(word, Frequency::from_str)(i)
 }
@@ -129,6 +123,8 @@ mod tests {
                     "rsdt_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_19910101-19951231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -153,6 +149,8 @@ mod tests {
                     "prw_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_19960101-20001231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -177,6 +175,8 @@ mod tests {
                     "ta500_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_20060101-20101231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -201,6 +201,8 @@ mod tests {
                     "tauu_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_19910101-19951231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -225,6 +227,8 @@ mod tests {
                     "rlut_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_20010101-20051231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -249,6 +253,8 @@ mod tests {
                     "ua500_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_19960101-20001231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -273,6 +279,8 @@ mod tests {
                     "tauv_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_20110101-20111130.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -297,6 +305,8 @@ mod tests {
                     "hus850_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_20110101-20111130.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -321,6 +331,8 @@ mod tests {
                     "evspsbl_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_20010101-20051231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -345,6 +357,8 @@ mod tests {
                     "va200_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_19900101-19901231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -369,6 +383,8 @@ mod tests {
                     "sfcWindmax_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_20110101-20111130.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",
@@ -393,6 +409,8 @@ mod tests {
                     "zmla_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_MOHC-HadRM3P_v1_day_20010101-20051231.nc"
                 ),
                 PathMetadata {
+                    activity: "cordex",
+                    product: "output",
                     domain: "AFR-44",
                     institution: "MOHC",
                     gcm_model_name: "ECMWF-ERAINT",