.gitlab-ci.yml

@@ -23,5 +23,7 @@ run_pre_commit_hooks:
     paths:
       - ${PRE_COMMIT_HOME}
   script:
+    - apk add zeromq-dev libffi-dev
+    - apk add --repository https://dl-cdn.alpinelinux.org/alpine/edge/main rust cargo
     - python3 -m pip install pre-commit
     - pre-commit run --all-files

dataset.py

+from collections.abc import Sequence
+
+
+class dataset_location:
+    def __init__(
+        self,
+        method: str,
+        host: str,
+        path: str,
+    ):
+        self.method = method
+        self.host = host
+        self.path = path
+
+    def is_valid(self) -> bool:
+        return all(x is not None for x in vars(self).values())
+
+    def serialize(self):
+        return vars(self)
+
+
+class dataset:
+    def __init__(
+        self, name: str, serve: bool, locations: Sequence[dataset_location], **kwargs
+    ):
+        self.name = name
+        self.serve = serve
+        self.locations = set(locations)
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+    def is_valid(self) -> bool:
+        return (
+            all(x is not None for x in vars(self).values())
+            and isinstance(self.serve, bool)
+            and all(type(x) is dataset_location for x in self.locations)
+            and all(x.is_valid() for x in self.locations)
+        )
+
+    def serialize(self):
+        rd = dict(self.vars())
+        rd["locations"] = set(x.serialize() for x in self.locations)
+        return rd
+
+    def from_dict(the_dict):
+        locations = set(dataset_location(**x) for x in the_dict["locations"])
+        return dataset(
+            name=the_dict["name"], serve=the_dict["serve"], locations=locations
+        )

test_dataset.py

+import dataset
+import unittest
+
+
+class test_dataset(unittest.TestCase):
+    def test_dataset(self):
+        dsl1 = dataset.dataset_location(method="direct", host="local", path="/scratch/")
+        ds = dataset.dataset(name="test_ds", serve=True, locations=(dsl1,))
+        self.assertTrue(ds.is_valid())
+
+        dsl2 = dataset.dataset_location(method="direct", host="local", path="/work/")
+        ds = dataset.dataset(name="test_ds", serve=True, locations=(dsl1, dsl2))
+        self.assertTrue(ds.is_valid())
+
+    def test_bad_dataset_location(self):
+        badhost = dataset.dataset_location(method="direct", host=None, path="/scratch/")
+        self.assertFalse(badhost.is_valid())
+
+        badpath = dataset.dataset_location(method="direct", host="local", path=None)
+        self.assertFalse(badpath.is_valid())
+
+        badmethod = dataset.dataset_location(method=None, host="local", path="/scratch")
+        self.assertFalse(badmethod.is_valid())
+
+    def test_bad_dataset(self):
+        dsl1 = dataset.dataset_location(method="direct", host="local", path="/scratch/")
+
+        bad_name = dataset.dataset(name=None, serve=True, locations=(dsl1,))
+        self.assertFalse(bad_name.is_valid())
+
+        bad_serve = dataset.dataset(name="test", serve=7, locations=(dsl1,))
+        self.assertFalse(bad_serve.is_valid())
+
+        with self.assertRaises(TypeError):
+            bad_location = dataset.dataset(name=None, serve=True, locations=dsl1)
+
+        badpath = dataset.dataset_location(method="direct", host="local", path=None)
+
+        bad_location = dataset.dataset(name=None, serve=True, locations=(badpath,))
+        self.assertFalse(bad_location.is_valid())
+
+        bad_location = dataset.dataset(name=None, serve=True, locations=(None,))
+        self.assertFalse(bad_location.is_valid())
+
+
+if __name__ == "__main__":
+    unittest.main()

utils.py

+from urllib.parse import urlparse
+
+
+def check_url(url):
+    result = urlparse(url)
+    if result.scheme and result.netloc:
+        return True
+    else:
+        return False