Ingest

Initial working version of ingestion. Originally put in another repo but recreated here since this is where the code will be hosted from now on. In order for this PR to work though, I had to manually copy things over which lost the old commit history though luckily there wasn't that much.

freva/src/drs/ingest.rs

@@ -9,7 +9,7 @@ use camino::Utf8PathBuf;
 use thiserror::Error;
 use tokio::sync::mpsc::{channel, Receiver};
 use tokio::time::Instant;
-use tracing::{error, info, trace, warn};
+use tracing::{error, info, instrument, trace, warn};
 use walkdir::{DirEntry, WalkDir};
 
 use crate::drs::Structure;
@@ -56,8 +56,8 @@ pub struct StructureReport {
 /// One task is the computation thread which processes the file names as they come in and it spawns the second task
 /// which is a blocking thread that uses `walkdir` synchronously and feeds the paths it finds into a channel.
 ///
-/// `files_collection` and `latest_collection` are optional. If `None`, then this will use Freva's defaults: `"files"`
-/// and `"latest"`, respectively.
+/// `files_collection` and `latest_collection` are optional. If `None`, then this will use Freva's defaults:
+/// [`DEFAULT_FILES_COLLECTION`] and [`DEFAULT_LATEST_COLLECTION`], respectively.
 pub async fn ingest<P: AsRef<Path>>(
     solr: &Solr,
     drs_conf: &Config,
@@ -126,6 +126,7 @@ pub async fn ingest<P: AsRef<Path>>(
 ///
 /// Invalid files, whatever the reason they're invalid, will be skipped. Failing to send to solr will result in this
 /// ending early with an error. Any files that were already sent to solr will not be cleaned up.
+#[instrument(skip(solr, structure), fields(structure = structure.activity().as_str()))]
 async fn ingest_structure(
     solr: Solr,
     structure: Structure,
@@ -175,8 +176,7 @@ async fn ingest_structure(
                 Ok(p) => p,
                 Err(e) => {
                     warn!(
-                        "{} not a valid drs file, has non UTF8 characters in path:\n{:?}",
-                        entry.path().display(),
+                        "not a valid drs file, has non UTF8 characters in path:\n{:?}",
                         e
                     );
                     skipped += 1;
@@ -187,7 +187,7 @@ async fn ingest_structure(
             let f = match structure.metadata_from_path(&path) {
                 Ok(f) => f,
                 Err(e) => {
-                    warn!("{} not a valid drs file, skipping:\n{:?}", path, e);
+                    warn!("not a valid drs file, skipping:\n{:#?}", e);
                     skipped += 1;
                     continue;
                 }