Merge branch 'master' of https://gitlab.dkrz.de/freva/deployment

.gitlab-ci.yml

@@ -12,7 +12,7 @@ lint:
         - python3 -m pip install .[test]
     script:
         - mypy --install-types --non-interactive
-        - black --check src
+        - black --check -t py310 src
 
 pages:
     image: python

assets/config/inventory.toml

@@ -17,13 +17,17 @@
 # The the project name that should be used for this freva instance
 # NOTE: this key has to be the first in the file
 project_name = ""
+
 ## Set the path to the SSL certfificate files that is used to make password
 ## queries to the vault server or used as web certfificates.
 [certificates]
+
 ## Path to the public keyfile
 public_keyfile = ""
+
 ## Path to the private keyfile
 privat_keyfile = ""
+
 ## Path to the chain file
 chain_keyfile = ""
 
@@ -63,19 +67,27 @@ hosts = ""
 ##in order to deploy the system correctly
 ##
 [db.config]
+
 ## Config variables for the database service
 port = 3306
 user = ""
 db = ""
+
 ##If you need a different user name you can set it here:
 #ansible_user = "username"
+
 ## You can set the db_host seperately, if none is given (default)
 ## the one from the hostsnames above are taken
-##db_host = ""
+
+db_host = ""
+
 ## Ansible needs a python3 interpreter, which can be set for custom python3 instances
 #ansible_python_interpreter = ""
+
 ##If you need a different user name you can set it here:
+
 #ansible_python_interpreter="/usr/bin/python3"
+
 ##Indicate whether or not to empty any pre-existing folders/docker volumes.
 ##(Useful for a truely fresh start) (default: False)
 wipe = false
@@ -83,12 +95,16 @@ wipe = false
 [solr.config]
 ## Config variables for the solr service
 port = 8983
+
 # Set the memory for the solr server
 mem = "4g"
 ## Ansible needs a python3 interpreter, which can be set for custom python3 instances
+
 #ansible_python_interpreter = ""
+
 ## If you need a different user name you can set it here:
 #ansible_user = "username"
+
 ##Indicate whether or not to empty any pre-existing folders/docker volumes.
 ##(Useful for a truely fresh start) (default: False)
 wipe = false
@@ -98,48 +114,62 @@ wipe = false
 ## List of user(s) that can alter the configuration of the freva cmd line interface
 ## If blank, the user that runs the deployment is chosen
 admins = ""
+
 ## The path where the core should be installed
 install_dir=""
+
 ## Set the path to any existing conda executable on the target machine,
 ## if not existing (default) a tmporary conda distribution will be downloaded
 conda_exec_path=""
+
 ## The directory where the project configuration files will be stored, leave
 ## blank to use the same directory as `install_dir`
 root_dir = ""
+
 ## If you which not to install a core instance but only configure one set the
 ## install variable to false. This can be useful if you have a central instance
 ## of freva deployed and want to setup a project specific configuration that
 ## uses this central instance
 install = true
+
 ## The directory where the user specific output will be stored
 base_dir_location = ""
+
 ## Set the directory holding the user content, like plots, for the web user
 ## interface. Note: after plugin application, display content of the plugin
 ## output will be copied to this directory. The default location of this
 ## directory (if left value left blank) is ${base_dir_location}/share/preview
 preview_path = ""
+
 ## Set the workload manager system, currently available are:
 ## "local", "slurm", "pbs", "lfs", "moab", "oar", "sge"
 scheduler_system = "local"
+
 ## Set the path to the directory that containes the stdout of the plugings,
 ## this directory must be accessible to the web ui. The workload manager
 ## will write the stdout into this directory. Defaults to ${base_dir_location}/share
 scheduler_output_dir = ""
+
 # Set the target architecutre of the system where the backend will be installed
 # to. You can choose from the following options:
 # Linux-x86_64 (default), Linux-aarch64, Linux-ppc64le, Linux-s390x, MacOSX-x86_64
 arch = "Linux-x86_64"
+
 ## If you need to install the core or its configuration as a different user you can
 ## set the ansible_become_user variable, this will install the the core as a
 ## different user
 #ansible_become_user = "username"
+
 ##If you need a different user name you can set it here:
 #ansible_user = "username"
+
 ## Ansible needs a python3 interpreter, which can be set for custom python3 instances
 #ansible_python_interpreter = ""
+
 ## The core deployment needs git, if git is not in the default PATH vraiable
 ## you can set the path to the git executable
 #git_path = ""
+
 ##Indicate whether or not to empty any pre-existing folders/docker volumes.
 ##(Useful for a truely fresh start) (default: False)
 wipe = false
@@ -148,69 +178,99 @@ wipe = false
 [web.config]
 ## List of user that can alter the configuration of freva web
 project_website = "www.freva.dkrz.de"
+
 ## Ansible needs a python3 interpreter, which can be set for custom python3 instances
 #ansible_python_interpreter = ""
+
 ##If you need a different user name you can set it here:
 #ansible_user = "username"
+
 ##Set html colors
 main_color = "Tomato"
 border_color = "#6c2e1f"
 hover_color = "#d0513a"
+
 ## The about us text is a small blurb about freva within the project
 about_us_text = ""
+
 ## Set the path to the institution logo, this should be the path to the logo
 ## as seen by the target system
 institution_logo = "/path/to/logo/on/target/machine"
-## Set a list of email addresses for contacts
-contacts = [""]
+
+## Set a an email addresses acting a a contact point for users
+contacts = ""
+
+## Set the smpt email server that will be used to send emails to contacts via the web ui
+email_host = ""
+
 ## Now set postal address
 imprint = "Project name, German Climate Computing Centre (DKRZ), Bundesstr. 45a, 20146 Hamburg, Germany."
+
 ## Here you can set a lengthy project description.
 ## You can also set a path to a filename that contains the information.
 ## Instead of text you can set a path to a file containing the text, like a html file.
 homepage_text = "Bal bla bla."
+
 ## Set a one line blurb of the project
 homepage_heading = "Short description of the project."
+
 ## Set the name of the project/institution
 institution_name = "Freva"
+
 ## Set the slurm scheduler host
 scheduler_host = ["levante.dkrz.de"]
+
 ## Settings for ldap
+
 ## Ldap server name(s)
 auth_ldap_server_uri = "ldap://idm-dmz.dkrz.de"
+
 ## Set the group that will be allowed to log on
 allowed_group = "test_group"
+
 ## Set the ldap search user base
 ldap_user_base = "cn=users,cn=accounts,dc=dkrz,dc=de"
+
 ## Set the ldap search group base, note: do not add the allowed_group as it will be auto added
 ldap_group_base = "cn=groups,cn=accounts,dc=dkrz,dc=de"
+
 ## distinguished name (dn) for the ldap user
 ldap_user_dn = "uid=dkrzagent,cn=sysaccounts,cn=etc,dc=dkrz,dc=de"
+
 ## use encrypted ldap connection (needs to be configured)
 auth_ldap_start_tls = false
+
 ## Set ldap last name search key
 ldap_last_name_field = "givenname"
+
 ## Set ldap first name search key
 ldap_first_name_field = "sn"
+
 ## Set ldap email earch key
 ldap_email_name_field = "mail"
-# Set the ldap group class name
+
+## Set the ldap group class name
 ldap_group_class = "groupOfNames"
-# Set the ldap group type, available values are are [posix, nested]
+
+## Set the ldap group type, available values are are [posix, nested]
 ldap_group_type = "nested"
-# Set the ldap tools class for users
-ldap_model = "MiklipUserInformation"
 
+## Set the ldap tools class for users
+ldap_model = "MiklipUserInformation"
 
 ## set the passwd for the ldap user
 ldap_user_pw = "dkrzprox"
+
 #######
 ## Set the hosts that are allowed to execute wsgi code
 allowed_hosts = ["www.freva.dkrz.de", "localhost"]
+
 ## Turn on/off debug mode on the website
 debug=false
+
 ## Which plugin id should be used for the web tour
 guest_tour_result = 105
+
 ## Set the menu entries
 # Menu entries consist of three entries these are:
 # [Label, url, html_id] -> e.g Plugins, plugins:home, plugin_menu

assets/playbooks/web-server-playbook.yml

 ---
+- hosts: vault
+  vars:
+    - ansible_python_interpreter: "{{ vault_ansible_python_interpreter }}"
+  tasks:
+    - name: Inserting email secrets
+      become: true
+      shell: >
+        /usr/local/bin/docker-or-podman exec -it {{ vault_name }}
+        vault kv put kv/email username={{vault_email_user}}
+        password='{{vault_email_password}}'
 - hosts: core
-
   vars:
     - ansible_python_interpreter: "{{ core_ansible_python_interpreter }}"
     - ansible_become_user: "{{ core_ansible_become_user if core_ansible_become_user is defined else 'root' }}"

assets/vault/runserver.py

@@ -85,8 +85,9 @@ class Vault(Resource):
         --------
             dict: status information
         """
+        out, status = {}, 401
         if public_key != os.environ["ROOT_PW"]:
-            return jsonify({"status": "fail"})
+            return out, status
         _, token = read_key()
         # Get the information from the vault
         url = f"http://127.0.0.1:8200/v1/kv/data/read-eval"
@@ -94,7 +95,7 @@ class Vault(Resource):
         out = requests.get(url, headers=headers).json()["data"]
         out["data"]["db.passwd"] = entry
         r = requests.post(url, json=out, headers=headers)
-        return jsonify({"status": "success"})
+        return {}, 201
 
     def get(self, entry, public_key):
         """Get method, for getting vault information.
@@ -114,15 +115,32 @@ class Vault(Resource):
         --------
             dict: Vault information
         """
-        out = ""
+        out = {}
+        status = 401
+        _, token = read_key()
+        if public_key != self.public_key:
+            return out, 401
+        if entry == "token":
+            return {"X-Vault-Token": token}, 200
         if entry == "data":
-            _, token = read_key()
             # Get the information from the vault
-            out = requests.get(
-                f"http://127.0.0.1:8200/v1/kv/data/read-eval",
+            req = requests.get(
+                "http://127.0.0.1:8200/v1/kv/data/read-eval",
+                headers={"X-Vault-Token": token},
+            )
+        else:
+            req = requests.get(
+                f"http://127.0.0.1:8200/v1/kv/data/{entry}",
                 headers={"X-Vault-Token": token},
-            ).json()["data"]["data"]
-        return jsonify(out)
+            )
+            out = req.json()["data"]["data"]
+            status = req.status_code
+        try:
+            out = req.json()["data"]["data"]
+            status = req.status_code
+        except KeyError:
+            out = req.json()
+        return out, status
 
 
 api.add_resource(Vault, "/vault/<entry>/<public_key>")  # Route_3

src/freva_deployment/deploy.py

@@ -20,6 +20,7 @@ from .utils import (
     asset_dir,
     config_dir,
     get_passwd,
+    get_email_credentials,
     logger,
     upload_server_map,
     RichConsole,
@@ -57,6 +58,7 @@ class DeployFactory:
 
         self._config_keys: list[str] = []
         self.master_pass: str = ""
+        self.email_password: str = ""
         self._td: TemporaryDirectory = TemporaryDirectory(prefix="inventory")
         self.inventory_file: Path = Path(self._td.name) / "inventory.yaml"
         self.eval_conf_file: Path = Path(self._td.name) / "evaluation_system.conf"
@@ -107,8 +109,8 @@ class DeployFactory:
         self.cfg["vault"]["config"]["root_passwd"] = self.master_pass
         self.cfg["vault"]["config"]["passwd"] = self.db_pass
         self.cfg["vault"]["config"]["keyfile"] = self.public_key_file
-        self.cfg["vault"]["config"]["email"] = ",".join(
-            self.cfg["web"]["config"].get("contacts", [])
+        self.cfg["vault"]["config"]["email"] = self.cfg["web"]["config"].get(
+            "contacts", ""
         )
 
     def _prep_db(self) -> None:
@@ -126,8 +128,8 @@ class DeployFactory:
         if not db_host:
             self.cfg["db"]["config"]["host"] = host
         self.cfg["db"]["config"].setdefault("port", "3306")
-        self.cfg["db"]["config"]["email"] = ",".join(
-            self.cfg["web"]["config"].get("contacts", [])
+        self.cfg["db"]["config"]["email"] = self.cfg["web"]["config"].get(
+            "contacts", ""
         )
         self._prep_vault()
 
@@ -139,8 +141,8 @@ class DeployFactory:
             self.cfg["solr"]["config"][key] = (
                 self.cfg["solr"]["config"].get(key) or default
             )
-        self.cfg["solr"]["config"]["email"] = ",".join(
-            self.cfg["web"]["config"].get("contacts", [])
+        self.cfg["solr"]["config"]["email"] = self.cfg["web"]["config"].get(
+            "contacts", ""
         )
 
     def _prep_core(self) -> None:
@@ -246,6 +248,13 @@ class DeployFactory:
             self.cfg[key]["config"]["config_toml_file"] = str(self.web_conf_file)
         if not self.master_pass:
             self.master_pass = get_passwd()
+        email_user, self.email_password = get_email_credentials()
+        self._prep_vault()
+        self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg["db"][
+            "config"
+        ].get("ansible_python_interpreter", "/usr/bin/python3")
+        self.cfg["vault"]["config"]["email_user"] = email_user
+        self.cfg["vault"]["config"]["email_password"] = self.email_password
         self.cfg["web"]["config"]["root_passwd"] = self.master_pass
         self.cfg["web"]["config"]["private_keyfile"] = self.private_key_file
         self.cfg["web"]["config"]["public_keyfile"] = self.public_key_file
@@ -474,13 +483,10 @@ class DeployFactory:
         self.create_eval_config()
         with self.inventory_file.open("w") as f_obj:
             f_obj.write(inventory)
-        if self.master_pass:
-            inventory_str = inventory.replace(
-                self.master_pass, "*" * len(self.master_pass)
-            )
-        else:
-            inventory_str = inventory
-        RichConsole.print(inventory_str, style="bold", markup=True)
+        for passwd in (self.email_password, self.master_pass):
+            if passwd:
+                inventory = inventory.replace(passwd, "*" * len(passwd))
+        RichConsole.print(inventory, style="bold", markup=True)
         logger.info("Playing the playbooks with ansible")
         RichConsole.print(
             "[b]Note:[/] The [blue]BECOME[/] password refers to the "

src/freva_deployment/ui/deployment_tui/deploy_forms.py

@@ -228,7 +228,7 @@ class WebScreen(BaseForm):
 
     def _add_widgets(self) -> None:
         """Add widgets to the screen."""
-        self.list_keys = "contacts", "imprint", "scheduler_host"
+        self.list_keys = "imprint", "scheduler_host"
         cfg = self.get_config(self.step)
         for key in self.list_keys:
             if key in cfg and isinstance(cfg[key], str):
@@ -298,9 +298,18 @@ class WebScreen(BaseForm):
                 self.add_widget_intelligent(
                     npyscreen.TitleText,
                     name=f"{self.num}Contact email address:",
-                    value=",".join(
-                        cast(List[str], cfg.get("contacts", "admin@freva.dkrz.de"))
+                    value=str(cfg.get("contacts", "data@dkrz.de")),
+                ),
+                True,
+            ),
+            email_host=(
+                self.add_widget_intelligent(
+                    npyscreen.TitleText,
+                    name=(
+                        f"{self.num}Smtp email that will be used to send "
+                        "emails to the contacts via the web ui"
                     ),
+                    value=cfg.get("email_host", "mailhost.dkrz.de"),
                 ),
                 True,
             ),

src/freva_deployment/utils.py

@@ -161,6 +161,25 @@ def upload_server_map(
         logger.error("Could not update server information %s", req.json())
 
 
+def get_email_credentials() -> tuple[str, str]:
+    """Read login credentials for email server.
+
+    Returns
+    =======
+    tuple: username and password
+    """
+
+    msg = (
+        "\nThe web will need login credentials to connect to the [b green]mail server [/]"
+        "that has been set up.\nYou should now enter your [it]login credentials[/].\n"
+        "[b]Note:[/]These credentials will be securely stored in an encrypted vault\n"
+    )
+    RichConsole.print(msg)
+    username = Prompt.ask("[green b]Username[/] for mail server")
+    password = Prompt.ask("[green b]Password[/] for mail server", password=True)
+    return username, password
+
+
 def get_passwd(min_characters: int = 8) -> str:
     """Create a secure pasword.