Ingest different DRS formats

This PR is a significant rewrite of how we handle DRS file paths to allow us to ingest paths of a few different known DRS formats.

This involved:

• creating a new crate to handle just loading the different DRS types
• changing ingestion to load the different types and then convert them to our metadata format
• fix some funky issues with data-dir not behaving how it should
  • I also made it optional so not having it means that all datasets are ingested

drs/src/parser/cmip6.rs

@@ -8,7 +8,7 @@ use nom::{
 };
 
 use crate::{
-    cmip6::{MemberId, PathMetadata, VariantLabel},
+    cmip6::{MemberId, PathMetadata, VariantLabel, MIP_ERA},
     parser::common::{name_segment, parse_temporal_subset, parse_u32, parse_version, path_segment},
 };
 
@@ -19,7 +19,7 @@ pub(crate) fn parse_cmip6(i: &str) -> IResult<&str, PathMetadata> {
     <mip_era>/<activity_id>/<institution_id>/<source_id>/<experiment_id>/
     <member_id>/<table_id>/<variable_id>/<grid_label>/<version>/<filename>
     */
-    let (i, mip_era) = terminated(parse_mip_era, path_sep)(i)?;
+    let (i, _mip_era) = terminated(parse_mip_era, path_sep)(i)?;
     let (i, activity_id) = path_segment(i)?;
     let (i, institution_id) = path_segment(i)?;
     let (i, source_id) = path_segment(i)?;
@@ -47,7 +47,6 @@ pub(crate) fn parse_cmip6(i: &str) -> IResult<&str, PathMetadata> {
     Ok((
         i,
         PathMetadata {
-            mip_era,
             activity_id,
             institution_id,
             source_id,
@@ -64,7 +63,7 @@ pub(crate) fn parse_cmip6(i: &str) -> IResult<&str, PathMetadata> {
 
 fn parse_mip_era(i: &str) -> IResult<&str, &str> {
     // get the segment and check that it is cmip6 which is required by the spec
-    verify(word, |w: &str| w == "CMIP6")(i)
+    verify(word, |w: &str| w == MIP_ERA)(i)
 }
 
 fn parse_member_id(i: &str) -> IResult<&str, MemberId> {
@@ -172,7 +171,6 @@ mod tests {
                     "tsl_Lmon_MPI-ESM-1-2-HAM_1pctCO2_r1i1p1f1_gn_191001-192912.nc"
                 ),
                 PathMetadata {
-                    mip_era: "CMIP6",
                     activity_id: "CMIP",
                     institution_id: "HAMMOZ-Consortium",
                     source_id: "MPI-ESM-1-2-HAM",
@@ -203,7 +201,6 @@ mod tests {
                     "c2h6_AERmon_BCC-ESM1_hist-piAer_r1i1p1f1_gn_185001-201412.nc"
                 ),
                 PathMetadata {
-                    mip_era: "CMIP6",
                     activity_id: "AerChemMIP",
                     institution_id: "BCC",
                     source_id: "BCC-ESM1",
@@ -234,7 +231,6 @@ mod tests {
                     "areacello_Ofx_CanESM5_G1_r3i1p2f1_gn.nc"
                 ),
                 PathMetadata {
-                    mip_era: "CMIP6",
                     activity_id: "GeoMIP",
                     institution_id: "CCCma",
                     source_id: "CanESM5",
@@ -261,7 +257,6 @@ mod tests {
                     "psl_6hrPlevPt_GFDL-CM4C192_highresSST-future_r1i1p1f1_gr3_203501010600-205101010000.nc"
                 ),
                 PathMetadata {
-                    mip_era: "CMIP6",
                     activity_id: "HighResMIP",
                     institution_id: "NOAA-GFDL",
                     source_id: "GFDL-CM4C192",
@@ -292,7 +287,6 @@ mod tests {
                     "sithick_SImon_MCM-UA-1-0_ssp370_r1i1p1f2_gn_201501-210012.nc"
                 ),
                 PathMetadata {
-                    mip_era: "CMIP6",
                     activity_id: "ScenarioMIP",
                     institution_id: "UA",
                     source_id: "MCM-UA-1-0",
@@ -323,7 +317,6 @@ mod tests {
                     "intpn2_Omon_CESM2_1pctCO2-cdr_r1i1p1f1_gr_005001-009912.nc"
                 ),
                 PathMetadata {
-                    mip_era: "CMIP6",
                     activity_id: "CDRMIP",
                     institution_id: "NCAR",
                     source_id: "CESM2",