Ingest

Initial working version of ingestion. Originally put in another repo but recreated here since this is where the code will be hosted from now on. In order for this PR to work though, I had to manually copy things over which lost the old commit history though luckily there wasn't that much.

freva/src/drs/ingest.rs

@@ -145,12 +145,18 @@ async fn ingest_structure(
         };
 
         if entry.path().is_file() {
-            // TODO: this conversion should probably take place within drs
-            let path = Utf8PathBuf::from_path_buf(entry.path().to_path_buf()).unwrap();
-            // TODO: this should too
-            if !structure.verify_path(&path) {
-                continue;
-            }
+            let path = match Utf8PathBuf::from_path_buf(entry.path().to_path_buf()) {
+                Ok(p) => p,
+                Err(e) => {
+                    warn!(
+                        "{} not a valid drs file, has non UTF8 characters in path:\n{:?}",
+                        entry.path().display(),
+                        e
+                    );
+                    skipped += 1;
+                    continue;
+                }
+            };
 
             let f = match structure.file_from_path(&path) {
                 Ok(f) => f,