diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a8a64b68984df4e3882dfa82b00d05163cd9ae2e..c3e2e33962f10cf8d4e5d77ea91ab7cf8a2582c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,7 +12,7 @@ lint:
- python3 -m pip install .[test]
script:
- mypy --install-types --non-interactive
- - black --check src
+ - black --check -t py310 src
pages:
image: python
diff --git a/assets/config/inventory.toml b/assets/config/inventory.toml
index a00512da776982e981021e19fd1ea8fcfd5f15ca..3204df39630c0bd0888678a7b6f160945e03ffdc 100644
--- a/assets/config/inventory.toml
+++ b/assets/config/inventory.toml
@@ -17,13 +17,17 @@
# The the project name that should be used for this freva instance
# NOTE: this key has to be the first in the file
project_name = ""
+
## Set the path to the SSL certfificate files that is used to make password
## queries to the vault server or used as web certfificates.
[certificates]
+
## Path to the public keyfile
public_keyfile = ""
+
## Path to the private keyfile
privat_keyfile = ""
+
## Path to the chain file
chain_keyfile = ""
@@ -63,19 +67,27 @@ hosts = ""
##in order to deploy the system correctly
##
[db.config]
+
## Config variables for the database service
port = 3306
user = ""
db = ""
+
##If you need a different user name you can set it here:
#ansible_user = "username"
+
## You can set the db_host seperately, if none is given (default)
## the one from the hostsnames above are taken
-##db_host = ""
+
+db_host = ""
+
## Ansible needs a python3 interpreter, which can be set for custom python3 instances
#ansible_python_interpreter = ""
+
##If you need a different user name you can set it here:
+
#ansible_python_interpreter="/usr/bin/python3"
+
##Indicate whether or not to empty any pre-existing folders/docker volumes.
##(Useful for a truely fresh start) (default: False)
wipe = false
@@ -83,12 +95,16 @@ wipe = false
[solr.config]
## Config variables for the solr service
port = 8983
+
# Set the memory for the solr server
mem = "4g"
## Ansible needs a python3 interpreter, which can be set for custom python3 instances
+
#ansible_python_interpreter = ""
+
## If you need a different user name you can set it here:
#ansible_user = "username"
+
##Indicate whether or not to empty any pre-existing folders/docker volumes.
##(Useful for a truely fresh start) (default: False)
wipe = false
@@ -98,48 +114,62 @@ wipe = false
## List of user(s) that can alter the configuration of the freva cmd line interface
## If blank, the user that runs the deployment is chosen
admins = ""
+
## The path where the core should be installed
install_dir=""
+
## Set the path to any existing conda executable on the target machine,
## if not existing (default) a tmporary conda distribution will be downloaded
conda_exec_path=""
+
## The directory where the project configuration files will be stored, leave
## blank to use the same directory as `install_dir`
root_dir = ""
+
## If you which not to install a core instance but only configure one set the
## install variable to false. This can be useful if you have a central instance
## of freva deployed and want to setup a project specific configuration that
## uses this central instance
install = true
+
## The directory where the user specific output will be stored
base_dir_location = ""
+
## Set the directory holding the user content, like plots, for the web user
## interface. Note: after plugin application, display content of the plugin
## output will be copied to this directory. The default location of this
## directory (if left value left blank) is ${base_dir_location}/share/preview
preview_path = ""
+
## Set the workload manager system, currently available are:
## "local", "slurm", "pbs", "lfs", "moab", "oar", "sge"
scheduler_system = "local"
+
## Set the path to the directory that containes the stdout of the plugings,
## this directory must be accessible to the web ui. The workload manager
## will write the stdout into this directory. Defaults to ${base_dir_location}/share
scheduler_output_dir = ""
+
# Set the target architecutre of the system where the backend will be installed
# to. You can choose from the following options:
# Linux-x86_64 (default), Linux-aarch64, Linux-ppc64le, Linux-s390x, MacOSX-x86_64
arch = "Linux-x86_64"
+
## If you need to install the core or its configuration as a different user you can
## set the ansible_become_user variable, this will install the the core as a
## different user
#ansible_become_user = "username"
+
##If you need a different user name you can set it here:
#ansible_user = "username"
+
## Ansible needs a python3 interpreter, which can be set for custom python3 instances
#ansible_python_interpreter = ""
+
## The core deployment needs git, if git is not in the default PATH vraiable
## you can set the path to the git executable
#git_path = ""
+
##Indicate whether or not to empty any pre-existing folders/docker volumes.
##(Useful for a truely fresh start) (default: False)
wipe = false
@@ -148,69 +178,99 @@ wipe = false
[web.config]
## List of user that can alter the configuration of freva web
project_website = "www.freva.dkrz.de"
+
## Ansible needs a python3 interpreter, which can be set for custom python3 instances
#ansible_python_interpreter = ""
+
##If you need a different user name you can set it here:
#ansible_user = "username"
+
##Set html colors
main_color = "Tomato"
border_color = "#6c2e1f"
hover_color = "#d0513a"
+
## The about us text is a small blurb about freva within the project
about_us_text = ""
+
## Set the path to the institution logo, this should be the path to the logo
## as seen by the target system
institution_logo = "/path/to/logo/on/target/machine"
-## Set a list of email addresses for contacts
-contacts = [""]
+
+## Set a an email addresses acting a a contact point for users
+contacts = ""
+
+## Set the smpt email server that will be used to send emails to contacts via the web ui
+email_host = ""
+
## Now set postal address
imprint = "Project name, German Climate Computing Centre (DKRZ), Bundesstr. 45a, 20146 Hamburg, Germany."
+
## Here you can set a lengthy project description.
## You can also set a path to a filename that contains the information.
## Instead of text you can set a path to a file containing the text, like a html file.
homepage_text = "Bal bla bla."
+
## Set a one line blurb of the project
homepage_heading = "Short description of the project."
+
## Set the name of the project/institution
institution_name = "Freva"
+
## Set the slurm scheduler host
scheduler_host = ["levante.dkrz.de"]
+
## Settings for ldap
+
## Ldap server name(s)
auth_ldap_server_uri = "ldap://idm-dmz.dkrz.de"
+
## Set the group that will be allowed to log on
allowed_group = "test_group"
+
## Set the ldap search user base
ldap_user_base = "cn=users,cn=accounts,dc=dkrz,dc=de"
+
## Set the ldap search group base, note: do not add the allowed_group as it will be auto added
ldap_group_base = "cn=groups,cn=accounts,dc=dkrz,dc=de"
+
## distinguished name (dn) for the ldap user
ldap_user_dn = "uid=dkrzagent,cn=sysaccounts,cn=etc,dc=dkrz,dc=de"
+
## use encrypted ldap connection (needs to be configured)
auth_ldap_start_tls = false
+
## Set ldap last name search key
ldap_last_name_field = "givenname"
+
## Set ldap first name search key
ldap_first_name_field = "sn"
+
## Set ldap email earch key
ldap_email_name_field = "mail"
-# Set the ldap group class name
+
+## Set the ldap group class name
ldap_group_class = "groupOfNames"
-# Set the ldap group type, available values are are [posix, nested]
+
+## Set the ldap group type, available values are are [posix, nested]
ldap_group_type = "nested"
-# Set the ldap tools class for users
-ldap_model = "MiklipUserInformation"
+## Set the ldap tools class for users
+ldap_model = "MiklipUserInformation"
## set the passwd for the ldap user
ldap_user_pw = "dkrzprox"
+
#######
## Set the hosts that are allowed to execute wsgi code
allowed_hosts = ["www.freva.dkrz.de", "localhost"]
+
## Turn on/off debug mode on the website
debug=false
+
## Which plugin id should be used for the web tour
guest_tour_result = 105
+
## Set the menu entries
# Menu entries consist of three entries these are:
# [Label, url, html_id] -> e.g Plugins, plugins:home, plugin_menu
diff --git a/assets/playbooks/web-server-playbook.yml b/assets/playbooks/web-server-playbook.yml
index 915832ac6a004ec575f2626d9fa08902d83ed3aa..bb68f053435f8284f228da18a5492c3131739799 100644
--- a/assets/playbooks/web-server-playbook.yml
+++ b/assets/playbooks/web-server-playbook.yml
@@ -1,6 +1,15 @@
---
+- hosts: vault
+ vars:
+ - ansible_python_interpreter: "{{ vault_ansible_python_interpreter }}"
+ tasks:
+ - name: Inserting email secrets
+ become: true
+ shell: >
+ /usr/local/bin/docker-or-podman exec -it {{ vault_name }}
+ vault kv put kv/email username={{vault_email_user}}
+ password='{{vault_email_password}}'
- hosts: core
-
vars:
- ansible_python_interpreter: "{{ core_ansible_python_interpreter }}"
- ansible_become_user: "{{ core_ansible_become_user if core_ansible_become_user is defined else 'root' }}"
diff --git a/assets/vault/runserver.py b/assets/vault/runserver.py
index fd16e744aaf93eeeda11005de6993ca634a2eeb9..3dd027bb7a100d078cc11614bc44a8fb59c58927 100644
--- a/assets/vault/runserver.py
+++ b/assets/vault/runserver.py
@@ -85,8 +85,9 @@ class Vault(Resource):
--------
dict: status information
"""
+ out, status = {}, 401
if public_key != os.environ["ROOT_PW"]:
- return jsonify({"status": "fail"})
+ return out, status
_, token = read_key()
# Get the information from the vault
url = f"http://127.0.0.1:8200/v1/kv/data/read-eval"
@@ -94,7 +95,7 @@ class Vault(Resource):
out = requests.get(url, headers=headers).json()["data"]
out["data"]["db.passwd"] = entry
r = requests.post(url, json=out, headers=headers)
- return jsonify({"status": "success"})
+ return {}, 201
def get(self, entry, public_key):
"""Get method, for getting vault information.
@@ -114,15 +115,32 @@ class Vault(Resource):
--------
dict: Vault information
"""
- out = ""
+ out = {}
+ status = 401
+ _, token = read_key()
+ if public_key != self.public_key:
+ return out, 401
+ if entry == "token":
+ return {"X-Vault-Token": token}, 200
if entry == "data":
- _, token = read_key()
# Get the information from the vault
- out = requests.get(
- f"http://127.0.0.1:8200/v1/kv/data/read-eval",
+ req = requests.get(
+ "http://127.0.0.1:8200/v1/kv/data/read-eval",
+ headers={"X-Vault-Token": token},
+ )
+ else:
+ req = requests.get(
+ f"http://127.0.0.1:8200/v1/kv/data/{entry}",
headers={"X-Vault-Token": token},
- ).json()["data"]["data"]
- return jsonify(out)
+ )
+ out = req.json()["data"]["data"]
+ status = req.status_code
+ try:
+ out = req.json()["data"]["data"]
+ status = req.status_code
+ except KeyError:
+ out = req.json()
+ return out, status
api.add_resource(Vault, "/vault/<entry>/<public_key>") # Route_3
diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py
index 571e422b5469eb7e47ae4cfca7929780da923347..1f1b1e91699643c00bd53e06c788906b3fb1cf5b 100644
--- a/src/freva_deployment/deploy.py
+++ b/src/freva_deployment/deploy.py
@@ -20,6 +20,7 @@ from .utils import (
asset_dir,
config_dir,
get_passwd,
+ get_email_credentials,
logger,
upload_server_map,
RichConsole,
@@ -57,6 +58,7 @@ class DeployFactory:
self._config_keys: list[str] = []
self.master_pass: str = ""
+ self.email_password: str = ""
self._td: TemporaryDirectory = TemporaryDirectory(prefix="inventory")
self.inventory_file: Path = Path(self._td.name) / "inventory.yaml"
self.eval_conf_file: Path = Path(self._td.name) / "evaluation_system.conf"
@@ -107,8 +109,8 @@ class DeployFactory:
self.cfg["vault"]["config"]["root_passwd"] = self.master_pass
self.cfg["vault"]["config"]["passwd"] = self.db_pass
self.cfg["vault"]["config"]["keyfile"] = self.public_key_file
- self.cfg["vault"]["config"]["email"] = ",".join(
- self.cfg["web"]["config"].get("contacts", [])
+ self.cfg["vault"]["config"]["email"] = self.cfg["web"]["config"].get(
+ "contacts", ""
)
def _prep_db(self) -> None:
@@ -126,8 +128,8 @@ class DeployFactory:
if not db_host:
self.cfg["db"]["config"]["host"] = host
self.cfg["db"]["config"].setdefault("port", "3306")
- self.cfg["db"]["config"]["email"] = ",".join(
- self.cfg["web"]["config"].get("contacts", [])
+ self.cfg["db"]["config"]["email"] = self.cfg["web"]["config"].get(
+ "contacts", ""
)
self._prep_vault()
@@ -139,8 +141,8 @@ class DeployFactory:
self.cfg["solr"]["config"][key] = (
self.cfg["solr"]["config"].get(key) or default
)
- self.cfg["solr"]["config"]["email"] = ",".join(
- self.cfg["web"]["config"].get("contacts", [])
+ self.cfg["solr"]["config"]["email"] = self.cfg["web"]["config"].get(
+ "contacts", ""
)
def _prep_core(self) -> None:
@@ -246,6 +248,13 @@ class DeployFactory:
self.cfg[key]["config"]["config_toml_file"] = str(self.web_conf_file)
if not self.master_pass:
self.master_pass = get_passwd()
+ email_user, self.email_password = get_email_credentials()
+ self._prep_vault()
+ self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg["db"][
+ "config"
+ ].get("ansible_python_interpreter", "/usr/bin/python3")
+ self.cfg["vault"]["config"]["email_user"] = email_user
+ self.cfg["vault"]["config"]["email_password"] = self.email_password
self.cfg["web"]["config"]["root_passwd"] = self.master_pass
self.cfg["web"]["config"]["private_keyfile"] = self.private_key_file
self.cfg["web"]["config"]["public_keyfile"] = self.public_key_file
@@ -474,13 +483,10 @@ class DeployFactory:
self.create_eval_config()
with self.inventory_file.open("w") as f_obj:
f_obj.write(inventory)
- if self.master_pass:
- inventory_str = inventory.replace(
- self.master_pass, "*" * len(self.master_pass)
- )
- else:
- inventory_str = inventory
- RichConsole.print(inventory_str, style="bold", markup=True)
+ for passwd in (self.email_password, self.master_pass):
+ if passwd:
+ inventory = inventory.replace(passwd, "*" * len(passwd))
+ RichConsole.print(inventory, style="bold", markup=True)
logger.info("Playing the playbooks with ansible")
RichConsole.print(
"[b]Note:[/] The [blue]BECOME[/] password refers to the "
diff --git a/src/freva_deployment/ui/deployment_tui/deploy_forms.py b/src/freva_deployment/ui/deployment_tui/deploy_forms.py
index 076fa0e122160ddc3cc21ac8457f032ce37bf192..7fa14df2397369e6cd3c22c7d10c41ba12a7b5a3 100644
--- a/src/freva_deployment/ui/deployment_tui/deploy_forms.py
+++ b/src/freva_deployment/ui/deployment_tui/deploy_forms.py
@@ -228,7 +228,7 @@ class WebScreen(BaseForm):
def _add_widgets(self) -> None:
"""Add widgets to the screen."""
- self.list_keys = "contacts", "imprint", "scheduler_host"
+ self.list_keys = "imprint", "scheduler_host"
cfg = self.get_config(self.step)
for key in self.list_keys:
if key in cfg and isinstance(cfg[key], str):
@@ -298,9 +298,18 @@ class WebScreen(BaseForm):
self.add_widget_intelligent(
npyscreen.TitleText,
name=f"{self.num}Contact email address:",
- value=",".join(
- cast(List[str], cfg.get("contacts", "admin@freva.dkrz.de"))
+ value=str(cfg.get("contacts", "data@dkrz.de")),
+ ),
+ True,
+ ),
+ email_host=(
+ self.add_widget_intelligent(
+ npyscreen.TitleText,
+ name=(
+ f"{self.num}Smtp email that will be used to send "
+ "emails to the contacts via the web ui"
),
+ value=cfg.get("email_host", "mailhost.dkrz.de"),
),
True,
),
diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py
index 77f10eb42e939f2a9879ea29037b5798fe22506f..c90fa91d44b9e794db72521dabdf299d729c270d 100644
--- a/src/freva_deployment/utils.py
+++ b/src/freva_deployment/utils.py
@@ -161,6 +161,25 @@ def upload_server_map(
logger.error("Could not update server information %s", req.json())
+def get_email_credentials() -> tuple[str, str]:
+ """Read login credentials for email server.
+
+ Returns
+ =======
+ tuple: username and password
+ """
+
+ msg = (
+ "\nThe web will need login credentials to connect to the [b green]mail server [/]"
+ "that has been set up.\nYou should now enter your [it]login credentials[/].\n"
+ "[b]Note:[/]These credentials will be securely stored in an encrypted vault\n"
+ )
+ RichConsole.print(msg)
+ username = Prompt.ask("[green b]Username[/] for mail server")
+ password = Prompt.ask("[green b]Password[/] for mail server", password=True)
+ return username, password
+
+
def get_passwd(min_characters: int = 8) -> str:
"""Create a secure pasword.