.gitlab-ci.yml

-# You can override the included template(s) by including variable overrides
-# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
-# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
-# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
-# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings
-# Note that environment variables can be set in several places
-# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
 stages:
 - test
-sast:
-  stage: test
-include:
-- template: Security/SAST.gitlab-ci.yml
-
 
 run_pre_commit_hooks:
   tags:
@@ -27,3 +15,5 @@ run_pre_commit_hooks:
     - apk add --repository https://dl-cdn.alpinelinux.org/alpine/edge/main rust cargo
     - python3 -m pip install pre-commit
     - pre-commit run --all-files
+    - pip install -r requirements.txt
+    - ./run_unit_tests

dataset.py

+from dataclasses import dataclass
+from serde import serde
+
+
+@serde
+@dataclass
+class dataset_location:
+    method: str
+    host: str
+    path: str
+
+    def is_valid(self) -> bool:
+        return all(x is not None for x in vars(self).values())
+
+    def serialize(self):
+        return vars(self)
+
+
+@serde
+@dataclass
+class dataset:
+    name: str
+    serve: bool
+    locations: list[dataset_location]
+
+    def __post_init__(self):
+        self.locations = list(self.locations)
+        if not all(isinstance(x, dataset_location) for x in self.locations):
+            raise TypeError(
+                f"Dateset locations must be of type dataset_location, found {[type(x) for x in self.locations]}"
+            )
+
+    def is_valid(self) -> bool:
+        return (
+            all(x is not None for x in vars(self).values())
+            and isinstance(self.serve, bool)
+            and all(type(x) is dataset_location for x in self.locations)
+            and all(x.is_valid() for x in self.locations)
+        )

experiment.py

+from dataclasses import dataclass
+from dataset import dataset
+from serde import serde
+
+
+@serde
+@dataclass
+class model_setup:
+    git_repo: str
+    commit_id: str
+
+
+@serde
+@dataclass
+class icon_setup(model_setup):
+    configure_command: str
+    config: str
+    host: str
+
+
+@serde
+@dataclass
+class experiment:
+    experiment_class: str
+    simulation_id: str
+    model: str
+    setup: icon_setup | model_setup
+    point_of_contact: str
+    copyright_info: str
+    cite_as: str
+    description: str
+    datasets: list[dataset]
+
+    def __post_init__(self):
+        self.datasets = list(self.datasets)
+        if not all(isinstance(x, dataset) for x in self.datasets):
+            raise TypeError(
+                f"Datesets must be of type dataset, found {[type(x) for x in self.datasets]}"
+            )
+        if not isinstance(self.setup, model_setup):
+            raise TypeError(
+                f"setup must be of type model_setup, found {type(self.setup)}."
+            )

requirements.txt

+pyserde

run_unit_tests

+#!/usr/bin/env sh
+
+export PYTHONPATH=$(pwd):$PYTHONPATH
+python -m unittest unit_tests/test_*.py
\ No newline at end of file

unit_tests/test_dataset.py

+import dataset
+import unittest
+from serde.yaml import from_yaml, to_yaml
+
+
+class test_dataset(unittest.TestCase):
+    dsl1 = dataset.dataset_location(method="direct", host="local", path="/scratch/")
+    dsl2 = dataset.dataset_location(method="direct", host="local", path="/work/")
+    ds = dataset.dataset(name="test_ds", serve=True, locations=(dsl1, dsl2))
+
+    def test_dataset(self):
+        dsl1 = test_dataset.dsl1
+        ds = dataset.dataset(name="test_ds", serve=True, locations=(dsl1,))
+        self.assertTrue(ds.is_valid())
+
+        ds = test_dataset.ds
+        self.assertTrue(ds.is_valid())
+
+    def test_bad_dataset_location(self):
+        badhost = dataset.dataset_location(method="direct", host=None, path="/scratch/")
+        self.assertFalse(badhost.is_valid())
+
+        badpath = dataset.dataset_location(method="direct", host="local", path=None)
+        self.assertFalse(badpath.is_valid())
+
+        badmethod = dataset.dataset_location(method=None, host="local", path="/scratch")
+        self.assertFalse(badmethod.is_valid())
+
+    def test_bad_dataset(self):
+        dsl1 = dataset.dataset_location(method="direct", host="local", path="/scratch/")
+
+        bad_name = dataset.dataset(name=None, serve=True, locations=(dsl1,))
+        self.assertFalse(bad_name.is_valid())
+
+        bad_serve = dataset.dataset(name="test", serve=7, locations=(dsl1,))
+        self.assertFalse(bad_serve.is_valid())
+
+        with self.assertRaises(TypeError):
+            bad_location = dataset.dataset(name=None, serve=True, locations=dsl1)
+
+        badpath = dataset.dataset_location(method="direct", host="local", path=None)
+
+        bad_location = dataset.dataset(name=None, serve=True, locations=(badpath,))
+        self.assertFalse(bad_location.is_valid())
+
+        with self.assertRaises(TypeError):
+            bad_location = dataset.dataset(name=None, serve=True, locations=(None,))
+
+    def test_to_from_yaml(self):
+        ds = test_dataset.ds
+        with open("test.yaml", "w") as of:
+            of.write(to_yaml(ds))
+        with open("test.yaml", "r") as infile:
+            obj = from_yaml(dataset.dataset, infile.read())
+            self.assertEqual(obj, ds)
+
+
+if __name__ == "__main__":
+    unittest.main()

unit_tests/test_experiment.py

+#!/usr/bin/env python
+
+import experiment
+import unittest
+from serde.yaml import from_yaml, to_yaml
+
+
+class test_experiment(unittest.TestCase):
+    setup = experiment.icon_setup(
+        configure_command="configure",
+        config="sensible",
+        host="levante",
+        git_repo="gitlab.dkrz.de",
+        commit_id="abc123",
+    )
+    experiment = experiment.experiment(
+        experiment_class="test",
+        simulation_id="test123",
+        model="icon",
+        setup=setup,
+        point_of_contact="flo",
+        copyright_info="cc-by-4.0 Flo",
+        cite_as="Flo",
+        description="test experiment",
+        datasets=[],
+    )
+
+    def test_experiment_fail(self):
+        with self.assertRaises(TypeError):
+            experiment.icon_setup(
+                configure_command="configure",
+                # config="sensible",
+                host="levante",
+                git_repo="gitlab.dkrz.de",
+                commit_id="abc123",
+            )
+
+        with self.assertRaises(TypeError):
+            experiment.experiment(
+                experiment_class="test",
+                simulation_id="test123",
+                model="icon",
+                setup="BAD SETUP",  # !
+                point_of_contact="flo",
+                copyright_info="cc-by-4.0 Flo",
+                cite_as="Flo",
+                description="test experiment",
+                datasets=[],
+            )
+
+        with self.assertRaises(TypeError):
+            experiment.experiment(
+                experiment_class="test",
+                simulation_id="test123",
+                model="icon",
+                setup=test_experiment.setup,
+                point_of_contact="flo",
+                copyright_info="cc-by-4.0 Flo",
+                cite_as="Flo",
+                description="test experiment",
+                datasets=("BAD", "DATASET"),  # !
+            )
+
+    def test_to_from_yaml(self):
+        test_exp = test_experiment.experiment
+        with open("test.yaml", "w") as of:
+            of.write(to_yaml(test_exp))
+        with open("test.yaml", "r") as infile:
+            obj = from_yaml(experiment.experiment, infile.read())
+            self.assertEqual(obj, test_exp)
+
+
+if __name__ == "__main__":
+    unittest.main()

utils.py

+from urllib.parse import urlparse
+
+
+def check_url(url):
+    result = urlparse(url)
+    if result.scheme and result.netloc:
+        return True
+    else:
+        return False