diff --git a/assets/config/inventory.toml b/assets/config/inventory.toml
index bbb3b62d7eb438791ce374f98c966138ebdc4337..f35e5b5384f589dc2679d39f5c73882a8314ea70 100644
--- a/assets/config/inventory.toml
+++ b/assets/config/inventory.toml
@@ -14,10 +14,16 @@
## The first part of this configuration defines general configuration that
## is common among all deployment steps.
-## Set the path to the public certfificate files that is used to make password
-## queries to the vault server or used as web certfificate. If you leave this
-## key empty a new certfificate will be created
-cert_file = ""
+## Set the path to the SSL certfificate files that is used to make password
+## queries to the vault server or used as web certfificates.
+[certificates]
+## Path to the public keyfile
+public_keyfile = ""
+## Path to the private keyfile
+privat_keyfile = ""
+## Path to the chain file
+chain_keyfile = ""
+
# The the project name that should be used for this freva instance
project_name = ""
@@ -104,10 +110,6 @@ root_dir = ""
## of freva deployed and want to setup a project specific configuration that
## uses this central instance
install = true
-## The url to the git repository of the evaluation_system
-git_url = "https://gitlab.dkrz.de/freva/evaluation_system.git"
-## Which branch of the evaluation_system should be deployed
-branch = "freva-dev"
## The directory where the user specific output will be stored
base_dir_location = ""
# Set the target architecutre of the system where the backend will be installed
@@ -133,16 +135,10 @@ wipe = false
[web.config]
## List of user that can alter the configuration of freva web
project_website = "www.freva.dkrz.de"
-## The url to the git repository of freva_web
-git_url = "https://gitlab.dkrz.de/freva/freva_web.git"
-## Which branch of the evaluation_system should be deployed
-branch = "master"
## Ansible needs a python3 interpreter, which can be set for custom python3 instances
#ansible_python_interpreter = ""
##If you need a different user name you can set it here:
#ansible_user = "username"
-##Set the path to the logo
-institution_logo = ""
##Set html colors
main_color = "Tomato"
border_color = "#6c2e1f"
@@ -153,23 +149,19 @@ about_us_text = ""
contacts = ""
## Now set postal address
address = "Project name, German Climate Computing Centre (DKRZ), Bundesstr. 45a, 20146 Hamburg, Germany."
-##Indicate whether or not to empty any pre-existing folders/docker volumes.
-##(Useful for a truely fresh start) (default: False)
-wipe = false
## Here you can set a lengthy project description.
## You can also set a path to a filename that contains the information.
## Instead of text you can set a path to a file containing the text, like a html file.
homepage_text = "Bal bla bla."
## Set a one line blurb of the project
-hompage_heading = "Short description of the project."
+homepage_heading = "Short description of the project."
## Set the name of the project/institution
institution_name = "Freva"
## Set the slurm scheduler host
scheduler_host = ["levante.dkrz.de"]
-
## Settings for ldap
## Ldap server name(s)
-auth_ldap_server_uri = "ldap://mldap0.hpc.dkrz.de,ldap://mldap1.hpc.dkrz.de"
+auth_ldap_server_uri = "ldap://idm-dmz.dkrz.de"
## Set the group that will be allowed to log on
allowed_group = "test_group"
## Set the ldap search user base
diff --git a/assets/db_service/Dockerfile b/assets/db_service/Dockerfile
deleted file mode 100644
index 90538ac373107595187f016a1353c4712e7e7417..0000000000000000000000000000000000000000
--- a/assets/db_service/Dockerfile
+++ /dev/null
@@ -1,11 +0,0 @@
-FROM mariadb:latest
-
-LABEL maintainer="DRKZ-CLINT"
-LABEL repository="https://gitlab.dkrz.de/freva/deployment.git"
-ENV NUM_BACKUPS=6
-ENV BACKUP_DIR=/var/lib/mysql/backup
-RUN apt -y update && apt -y install python3 pip
-COPY daily_backup.sh /usr/local/bin/daily_backup
-COPY password_rotate.py /usr/local/bin/pwassword_rotate
-RUN chmod +x /usr/local/bin/daily_backup /usr/local/bin/pwassword_rotate &&\
- python3 -m pip install requests
diff --git a/assets/db_service/daily_backup.sh b/assets/db_service/daily_backup.sh
deleted file mode 100644
index c62488032e9bd97817f661532b961c7433a6d2bd..0000000000000000000000000000000000000000
--- a/assets/db_service/daily_backup.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-###############################################
-# CREATE a daily backup of the freva database
-mkdir -p ${BACKUP_DIR}
-backup_f=${BACKUP_DIR}/backup-$(date +%Y%m%d_%H%M%S).sql.gz
-files_to_keep=$(ls -t ${BACKUP_DIR}/backup-*.sql.gz |head -n ${NUM_BACKUPS})
-for file in $(ls ${BACKUP_DIR}/backup-*.sql.gz);do
- is_new_file=$(echo ${files_to_keep} |grep $file)
- if [ "x${is_new_file}" = "x" ];then
- rm $file
- fi
-done
-mysqldump -u root -h localhost -p"${MYSQL_ROOT_PASSWORD}" --all-databases | gzip -c -9 -q > $backup_f
-chown mysql:mysql ${backup_f}
diff --git a/assets/playbooks/core-server-playbook.yml b/assets/playbooks/core-server-playbook.yml
index f72ebc69b8eef7f680422a218536c242f601b6c9..39cb50daff82ac4dcce3d99f41cf86318052d2f4 100644
--- a/assets/playbooks/core-server-playbook.yml
+++ b/assets/playbooks/core-server-playbook.yml
@@ -20,7 +20,6 @@
git:
repo: "{{ core_git_url }}"
dest: /tmp/evaluation_system
- version: "{{ core_branch }}"
executable: "{{core_git_path}}"
become: "{{'true' if core_ansible_become_user is defined else 'no' }}"
- name: Preparing creation of directory structure
@@ -108,12 +107,6 @@
PYTHON3: "{{ ansible_python_interpreter }}"
EVALUATION_SYSTEM_CONFIG_FILE: "{{ core_root_dir }}/{{eval_path}}"
become: "{{'yes' if core_ansible_become_user is defined else 'no' }}"
- - name: Copying Private key files
- copy:
- src: "{{core_private_keyfile}}"
- dest: "{{ core_root_dir }}/freva/{{ project_name }}.key"
- mode: "2644"
- become: "{{'yes' if core_ansible_become_user is defined else 'no' }}"
- name: Copying Public key file
copy:
src: "{{ core_keyfile }}"
diff --git a/assets/playbooks/db-server-playbook.yml b/assets/playbooks/db-server-playbook.yml
index 5d772f2343e9e18f2dac749d133b663a2ecd3174..7c2a1a8c2f0a47af07e18f04950ee956146128fb 100644
--- a/assets/playbooks/db-server-playbook.yml
+++ b/assets/playbooks/db-server-playbook.yml
@@ -5,11 +5,18 @@
- ansible_python_interpreter: "{{ db_ansible_python_interpreter }}"
- docker_cmd: >
--net {{ project_name }} -v
- /mnt/{{project_name}}/db_service:/var/lib/mysql:z
+ /opt/freva/{{project_name}}/db_service:/var/lib/mysql:z
--name {{ db_name }} -e HOST={{ db_host }}
- -e PROJECT={{ project_name }} -e DB_USER={{ db_user }}
+ -e NUM_BACKUPS=7
+ -e PROJECT={{ project_name }}
+ -e MYSQL_USER={{db_user}}
+ -e MYSQL_PASSWORD='{{db_passwd}}'
+ -e MYSQL_DATABASE={{db_db}}
+ -p {{ db_port }}:3306
-e MYSQL_ROOT_PASSWORD={{ root_passwd }}
- -p {{ db_port }}:3306 {{ db_name }}:latest
+ -v /root/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z
+ -v /root/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z
+ -t mariadb:latest
- continer_name: "{{ db_name }}"
- vault_name: "{{project_name}}-vault"
tasks:
@@ -23,11 +30,15 @@
stat:
path: /usr/bin/systemctl
register: systemctl
+ - name: Registering anacron path
+ stat:
+ path: /etc/cron.daily
+ register: cron
- name: Stopping services and deleting existing containers
shell: >
+ /usr/local/bin/docker-or-podman stop {{db_name}};
systemctl stop {{db_name}};
/usr/local/bin/docker-or-podman rm {{db_name}};
- /usr/local/bin/docker-or-podman rmi -f {{db_name}};
echo 0
become: true
ignore_errors: true
@@ -38,13 +49,12 @@
become: true
- name: Cleaning existing directory structure
file:
- path: /mnt/{{ project_name }}/db_service
+ path: /opt/freva/{{ project_name }}/db_service
state: absent
become: true
- when: wipe == true
- name: Creating directory structure
file:
- path: /mnt/{{ project_name }}/db_service
+ path: /opt/freva/{{ project_name }}/db_service
state: directory
recurse: true
group: 999
@@ -54,11 +64,6 @@
copy: src="{{ asset_dir }}/db_service" dest=/tmp
- name: Copying cron create script to target machine
copy: src="{{ asset_dir }}/scripts/create_cron.sh" dest=/tmp/db_service/
- - name: Building mariadb docker image
- shell:
- chdir: /tmp/db_service
- cmd: /usr/local/bin/docker-or-podman build -t {{ db_name }}:latest .
- become: true
- name: Copying systemd files
copy:
src: "{{ asset_dir }}/scripts/{{ item }}"
@@ -66,66 +71,30 @@
mode: '0755'
with_items:
- "create_systemd.py"
- - "dump_sql"
- - name: Running docker container
- become: true
+ - name: Getting additional configurations
+ become: true
+ git:
+ repo: https://gitlab.dkrz.de/freva/freva-service-config.git
+ dest: /root/freva-service-config
+ version: init
+ update: true
+ - name: Creating the mysql docker container
shell: /usr/local/bin/docker-or-podman run -d {{docker_cmd}}
- - name: Preparing root password reset
- shell: >
- echo "USE mysql; FLUSH PRIVILEGES; ALTER USER "
- "'root'@'localhost' IDENTIFIED BY '{{root_passwd}}'; "
- "ALTER USER 'root'@'%' IDENTIFIED BY '{{root_passwd}}'; "
- "FLUSH PRIVILEGES;" > /tmp/dump.sql
- - name: Copying sql script into docker container
- shell: >
- /usr/local/bin/docker-or-podman cp "{{item}}" {{db_name}}:"{{item}}"
- with_items:
- - "/tmp/dump.sql"
- - "/tmp/dump_sql"
- become: true
- - pause: seconds=5
- - name: Resetting the password
- shell: /usr/local/bin/docker-or-podman exec -it {{db_name}} /tmp/dump_sql
become: true
- ignore_errors: true
- - name: Deleting existing contianer
- shell: >
- /usr/local/bin/docker-or-podman stop "{{ db_name }}";
- /usr/local/bin/docker-or-podman rm "{{ db_name }}";
- echo 0
- become: true
- - name: Creating the mariadb docker container
- become: true
- shell: /usr/local/bin/docker-or-podman run -d {{docker_cmd}}
- name: Creating systemd service
become: true
shell: >
/tmp/create_systemd.py {{db_name}} --requires {{vault_name}} --enable
when: systemctl.stat.exists == true
- - pause: seconds=5
- - name: Copying sample data to target machine
- # libselinux-python3 needs to be installed for this
- copy: src={{ db_dump }} dest=/tmp/dump.sql
- - name: Adding sql data to docker container
- shell: >
- /usr/local/bin/docker-or-podman cp "{{item}}" {{db_name}}:"{{item}}"
- with_items:
- - "/tmp/dump.sql"
- - "/tmp/dump_sql"
- become: true
- - name: Inserting sample data into database
- shell: /usr/local/bin/docker-or-podman exec -it {{db_name}} /tmp/dump_sql
- become: true
- name: Creating cron jobs
become: true
shell: sh /tmp/db_service/create_cron.sh "{{ db_name }}" "{{db_email}}"
+ when: cron.stat.exists == true
- name: Deleting auxillary files
file:
state: absent
path: "{{ item }}"
with_items:
- - /tmp/dump.sql
- - /tmp/dump_sql
- /tmp/create_systemd.py
- /tmp/db_service
- name: Restarting docker container
diff --git a/assets/playbooks/solr-server-playbook.yml b/assets/playbooks/solr-server-playbook.yml
index 91a7585358858ed672d2773700e4d2c2d6bd9d82..19906efe9e1863e759a71adb3d4d48ce08d290f9 100644
--- a/assets/playbooks/solr-server-playbook.yml
+++ b/assets/playbooks/solr-server-playbook.yml
@@ -8,9 +8,13 @@
-e CORE=files
-e NUM_BACKUPS=7
-e SOLR_HEAP={{solr_mem}}
- -v /mnt/{{project_name}}/solr_service:/var/solr/data:z
+ -v /opt/freva/{{project_name}}/solr_service:/var/solr/data:z
+ -v /root/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml
+ -v /root/freva-service-config/docker-entrypoint-initdb.d/create_cores.sh
+ -v /root/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup
--name {{ solr_name }}
- -p {{ solr_port }}:8983 -t {{ solr_name }}:latest
+ -p {{ solr_port }}:8983 -t
+ -t solr:latest
tasks:
- name: Copying docker/podman wrapper script
copy:
@@ -22,6 +26,10 @@
stat:
path: /usr/bin/systemctl
register: systemctl
+ - name: Registering anacron path
+ stat:
+ path: /etc/cron.daily
+ register: cron
- name: Stopping services and deleting existing containers
shell: >
systemctl stop {{solr_name}};
@@ -39,32 +47,32 @@
become: true
- name: Cleaning existing directory structure
file:
- path: /mnt/{{ project_name }}/solr_service
+ path: /opt/freva/{{ project_name }}/solr_service
state: absent
become: true
when: wipe == true
- name: Creating directory structure
file:
- path: /mnt/{{ project_name }}/solr_service
+ path: /opt/freva/{{ project_name }}/solr_service
state: directory
owner: 8983
group: 8983
recurse: true
become: true
+ - name: Getting additional configurations
+ become: true
+ git:
+ repo: https://gitlab.dkrz.de/freva/freva-service-config.git
+ dest: /root/freva-service-config
+ version: init
+ update: true
- name: Copy systemd files
copy:
src: "{{ asset_dir }}/scripts/create_systemd.py"
dest: /tmp/create_systemd.py
mode: "0755"
- - name: Copy auxillary files to target machine
- copy: src="{{ asset_dir }}/solr_service" dest=/tmp
- name: Copy cron create script to target machine
copy: src="{{ asset_dir }}/scripts/create_cron.sh" dest=/tmp/solr_service/
- - name: Building solr images
- shell:
- chdir: /tmp/solr_service
- cmd: /usr/local/bin/docker-or-podman build -t {{ solr_name }}:latest .
- become: true
- name: Creating the solr docker container
become: true
shell: /usr/local/bin/docker-or-podman run -d {{docker_cmd}}
@@ -73,26 +81,12 @@
shell: /tmp/create_systemd.py {{solr_name}} --enable
when: systemctl.stat.exists == true
- pause: seconds=3
- - name: Creating the solr cores
- become: true
- shell: >
- /usr/local/bin/docker-or-podman exec -it --user=solr "{{solr_name}}"
- /bin/bash create_core.sh "{{item}}"
- with_items:
- - "latest"
- - "files"
- - name: Restarting docker container
- become: true
- shell: systemctl restart "{{ solr_name }}"
- when: systemctl.stat.exists == true
- name: Creating cron jobs
become: true
shell: >
sh /tmp/solr_service/create_cron.sh "{{ solr_name }}" "{{solr_email}}"
+ when: systemctl.stat.exists == true
- name: Deleting auxillary files
file:
state: absent
- path: "{{ item }}"
- with_items:
- - /tmp/create_systemd.py
- - /tmp/solr_service
+ path: /tmp/create_systemd.py
diff --git a/assets/playbooks/vault-server-playbook.yml b/assets/playbooks/vault-server-playbook.yml
index 602c96a97941006432e942082369bd116003aa4b..0550f3004d776dbbc41170be243fa8f236dc7f43 100644
--- a/assets/playbooks/vault-server-playbook.yml
+++ b/assets/playbooks/vault-server-playbook.yml
@@ -6,6 +6,8 @@
- continer_name: "{{ vault_name }}"
- docker_cmd: >
--net {{ project_name }} --cap-add=IPC_LOCK
+ -v /opt/freva/{{project_name}}/vault_service/certs:/data:z
+ -v /opt/freva/{{project_name}}/vault_service/vault:/vault/file:z
-p 5002:5002 --name={{ vault_name }} {{ vault_name }}:latest
tasks:
- name: Copying docker/podman wrapper script
@@ -18,6 +20,13 @@
stat:
path: /usr/bin/systemctl
register: systemctl
+ - name: Registering vault volume path
+ stat:
+ path: /usr/bin/systemctl
+ register: vault_volume
+ - name: Set vault valume path
+ set_fact:
+ exists: vault_volume.stat.exists
- name: Stopping services and deleting existing containers
shell: >
systemctl stop {{vault_name}};
@@ -34,14 +43,6 @@
become: true
- name: Copying auxillary files to target machine
copy: src="{{ asset_dir }}/vault" dest=/tmp
- - name: Copying public key file to target machine
- copy:
- src: "{{ vault_keyfile }}"
- dest: "/tmp/vault/{{ project_name }}.crt"
- - name: Copying private key file to target machine
- copy:
- src: "{{ vault_private_keyfile }}"
- dest: "/tmp/vault/{{ project_name }}.key"
- name: Copying systemd files
copy:
src: "{{ asset_dir }}/scripts/create_systemd.py"
@@ -49,14 +50,22 @@
mode: "0755"
- name: Cleaning existing directory structure
file:
- path: /mnt/{{ project_name }}/vault_service
+ path: /opt/freva/{{ project_name }}/vault_service/
state: absent
become: true
when: wipe == true
- name: Creating directory structure
file:
- path: /mnt/{{ project_name }}/vault_service
state: directory
+ path: "{{ item }}"
+ with_items:
+ - "/opt/freva/{{ project_name }}/vault_service/vault"
+ - "/opt/freva/{{ project_name }}/vault_service/certs"
+ become: true
+ - name: Copying public key file to target machine
+ copy:
+ src: "{{ vault_keyfile }}"
+ dest: "/opt/freva/{{project_name}}/vault_service/certs/freva.crt"
become: true
- name: Building vault images
shell:
@@ -76,16 +85,21 @@
when: systemctl.stat.exists == true
ignore_errors: true
- pause: seconds=3
- - name: Inserting secrets
+ - name: Inserting server infrastructure
+ become: true
shell: >
/usr/local/bin/docker-or-podman exec -it {{ vault_name }}
vault kv put kv/read-eval db.container={{ vault_name }}
- db.passwd='{{ vault_passwd }}'
db.host={{ vault_host }}
+ db.port={{ db_port }}
+ - name: Inserting secrets
+ shell: >
+ /usr/local/bin/docker-or-podman exec -it {{ vault_name }}
+ vault kv put kv/read-eval db.passwd='{{ vault_passwd }}'
db.user={{ db_user }}
db.db={{ db_db }}
- db.port={{ db_port }}
become: true
+ when: wipe == true or exists == false
- name: Deleting auxillary files
file:
state: absent
diff --git a/assets/playbooks/web-server-playbook.yml b/assets/playbooks/web-server-playbook.yml
index 11ea6c37605f7d33cebaf948e25019457ccdc3ec..1322a6f1b760cdb8de20ea33c0ba1fd7effeb9ac 100644
--- a/assets/playbooks/web-server-playbook.yml
+++ b/assets/playbooks/web-server-playbook.yml
@@ -5,16 +5,11 @@
- ansible_python_interpreter: "{{ core_ansible_python_interpreter }}"
- ansible_become_user: "{{ core_ansible_become_user if core_ansible_become_user is defined else 'root' }}"
tasks:
- - name: Getting register to the freva logo
- stat:
- path: "{{ core_institution_logo }}"
- register: logo
- name: Creating share directory with admin group
file:
path: "{{core_root_dir}}/freva/web"
state: directory
mode: "2775"
- recurse: true
group: "{{core_admin_group}}"
become: "{{'yes' if core_ansible_become_user is defined else 'no' }}"
when: core_admin_group is defined
@@ -26,13 +21,6 @@
recurse: true
become: "{{'yes' if core_ansible_become_user is defined else 'no' }}"
when: core_admin_group is not defined
- - name: Copy the logo if exists
- copy:
- src: "{{ core_institution_logo }}"
- dest: "{{core_root_dir}}/freva/web/logo.{{web_institution_logo_suffix}}"
- mode: "2775"
- become: "{{'yes' if core_ansible_become_user is defined else 'no' }}"
- when: logo.stat.exists == true
- name: Adding freva web config to the core structure
copy:
src: "{{ core_config_toml_file }}"
@@ -44,14 +32,22 @@
vars:
- ansible_python_interpreter: "{{ web_ansible_python_interpreter }}"
- - docker_cmd: >
- -p 8000:8000 -p 80:80 -p 443:443
- -v /mnt/{{project_name}}/web_service:/srv/http:z
+ - service_dir: /opt/freva/{{project_name}}/web_service
+ - docker_web_cmd: >
+ -p 8000:8000
-v {{ core_root_dir }}:{{ core_root_dir }}:ro
-v {{ core_base_dir_location }}:{{ core_base_dir_location }}:ro
- -v {{ core_base_dir_location }}/share/preview:/srv/http/static/preview:ro
- --name {{ web_name}} -t {{ web_name}}:latest
+ -e EVALUATION_SYSTEM_CONFIG_FILE={{core_root_dir}}/freva/evaluation_system.conf
+ --name {{web_name}}
+ -t registry.gitlab.dkrz.de/freva/freva_web/freva_web:production_dockers
+ - docker_apache_cmd: >
+ -v /opt/freva/{{project_name}}/web_service/freva_web.conf:/usr/local/apache2/conf/httpd.conf:z
+ -v /opt/freva/{{project_name}}/web_service/server-cert.crt:/etc/ssl/certs/server-cert.crt:z
+ -v /opt/freva/{{project_name}}/web_service/server-key.key:/etc/ssl/private/server-key.key:z
+ -v /opt/freva/{{project_name}}/web_service/cacert.pem:/etc/ssl/certs/cacert.pem:z
+ -e FREVA_HOST={{web_server_name}} -p 80:80 -p 443:443 httpd:latest
- redis_name: "{{ project_name }}-redis"
+ - apache_name: "{{project_name}}-apache"
tasks:
- name: Copying docker/podman wrapper script
copy:
@@ -78,68 +74,61 @@
shell: >
systemctl stop {{ web_name }};
systemctl stop {{ redis_name }};
+ systemctl stop {{ apache_name }};
/usr/local/bin/docker-or-podman stop {{web_name}};
/usr/local/bin/docker-or-podman rm {{web_name}};
- /usr/local/bin/docker-or-podman rmi -f {{web_name}};
/usr/local/bin/docker-or-podman stop {{redis_name}};
/usr/local/bin/docker-or-podman rm {{redis_name}};
+ /usr/local/bin/docker-or-podman stop {{apache_name}};
+ /usr/local/bin/docker-or-podman rm {{apache_name}};
echo 0
ignore_errors: true
become: true
-# - name: Creating docker network
-# shell: >
-# /usr/local/bin/docker-or-podman network create "{{ project_name }}";
-# echo 0
-# become: true
- name: Deleting existing web-directory
file:
state: absent
- path: /mnt/{{ project_name }}/web_service
+ path: "{{service_dir}}"
+ become: true
+ - name: Creating volume dir for web
+ become: true
+ file:
+ path: "{{service_dir}}"
+ state: directory
+ - name: Copying key files files
become: true
- when: wipe == true
- - name: Copy auxillary files
copy:
- src: "{{ asset_dir }}/web"
- dest: "/tmp/{{project_name}}_web"
- mode: "0755"
- - name: Adjusting freva_web.conf
- shell:
- chdir: "/tmp/{{project_name}}_web/web"
- cmd: >
- {{ansible_python_interpreter}} create_web_conf.py
- freva_web.conf freva_web.conf --website={{web_project_website}}
- --root-dir={{core_root_dir}} --project-name={{project_name}}
- --alias={{web_server_alias}} --server-name={{web_host}}
- --work-dir={{core_base_dir_location}}
+ src: "{{item.src}}"
+ dest: "{{item.dest}}"
+ with_items:
+ - {src: "{{asset_dir}}/web/freva_web.conf",
+ dest: "{{service_dir}}/freva_web.conf"}
+ - {src: "{{web_public_keyfile}}",
+ dest: "{{service_dir}}/server-cert.crt"}
+ - {src: "{{web_private_keyfile}}",
+ dest: "{{service_dir}}/server-key.key"}
+ - {src: "{{web_chain_keyfile}}",
+ dest: "{{service_dir}}/cacert.pem"}
+ - {src: "{{asset_dir}}/web/setup_web.sh",
+ dest: "{{service_dir}}/setup_web.sh"}
- name: Creating redis container
become: true
shell:
cmd: >
- /usr/local/bin/docker-or-podman pull redis:latest &&
/usr/local/bin/docker-or-podman run -d --name {{redis_name}}
-p 6379:6379 redis:latest
- - name: Building freva_web image
+ - name: Creating apache reverse proxy container
become: true
shell:
- chdir: "/tmp/{{project_name}}_web/web"
cmd: >
- /usr/local/bin/docker-or-podman build --username {{ansible_user}}
- --build-arg=core_url={{core_git_url}}
- --build-arg=core_branch={{core_branch}}
- --build-arg=build_root=/mnt/{{project_name}}/web_service
- --build-arg=git_url={{web_git_url}}
- --build-arg=git_branch={{web_branch}}
- --build-arg=root_dir={{core_root_dir}}
- --build-arg=PROJECT_NAME={{project_name}}
- -t {{ web_name }} .
- - name: Creating volume dir for web
- become: true
- file:
- path: /mnt/{{project_name}}/web_service
- state: directory
- - name: Running freva_web image
+ /usr/local/bin/docker-or-podman run -d --name {{apache_name}}
+ {{docker_apache_cmd}}
+ - name: debug
+ shell:
+ cmd: >
+ echo /usr/local/bin/docker-or-podman run -d --name {{apache_name}} {{docker_apache_cmd}} > /tmp/podman_cmd
+ - name: Getting freva_web image
become: true
- shell: /usr/local/bin/docker-or-podman run -d {{docker_cmd}}
+ shell: /usr/local/bin/docker-or-podman run -d {{docker_web_cmd}}
- name: Copying systemd files
copy:
src: "{{ asset_dir }}/scripts/create_systemd.py"
@@ -149,10 +138,16 @@
become: true
shell: |
/tmp/create_systemd.py {{redis_name}}
- /tmp/create_systemd.py {{web_name}} --requires {{redis_name}} --enable
+ /tmp/create_systemd.py {{apache_name}}
+ /tmp/create_systemd.py {{web_name}} --requires {{redis_name}} {{apache_name}} --enable
when: systemctl.stat.exists == true
- pause: seconds=5
- - name: Building django page
+ - name: Adding setup_web script
+ become: true
+ shell: >
+ /usr/local/bin/docker-or-podman cp {{service_dir}}/setup_web.sh
+ {{web_name}}:/tmp/setup_web.sh
+ - name: Setting django admin
become: true
shell: >
/usr/local/bin/docker-or-podman exec -it {{web_name}}
diff --git a/assets/scripts/create_cron.sh b/assets/scripts/create_cron.sh
index 48e670aefc2a80bf7c5a51a76e0398d81359cf52..9bcc97121e282dc46bece87d34b7b5dccee1f762 100755
--- a/assets/scripts/create_cron.sh
+++ b/assets/scripts/create_cron.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-cmd="/usr/local/bin/docker-or-podman exec $1 /usr/local/bin/daily_backup 1> /dev/null"
+cmd="/usr/local/bin/docker-or-podman exec $1 bash /usr/local/bin/daily_backup 1> /dev/null"
mkdir -p /etc/cron.daily
echo "#!/bin/sh" > /etc/cron.daily/backup_$1
echo "# Run daily backup for $1" >> /etc/cron.daily/backup_$1
diff --git a/assets/solr_service/Dockerfile b/assets/solr_service/Dockerfile
deleted file mode 100644
index 844816d20ebac401fff40d9172e194a8e6b9e163..0000000000000000000000000000000000000000
--- a/assets/solr_service/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-FROM solr:latest
-
-USER root
-LABEL maintainer="DRKZ-CLINT"
-LABEL repository="https://gitlab.dkrz.de/freva/deployment.git"
-ENV NUM_BACKUPS=7
-ENV CORE="files"
-COPY daily_backup.sh /usr/local/bin/daily_backup
-
-RUN echo "nameserver 8.8.8.8" >> /etc/resolv.conf &&\
- chmod +x /usr/local/bin/daily_backup
-VOLUME /var/solr/data
-USER solr
-COPY create_core.sh managed-schema.xml ./
diff --git a/assets/solr_service/create_core.sh b/assets/solr_service/create_core.sh
deleted file mode 100755
index 46a31b83dae72e9fc068fad23634506f275dce42..0000000000000000000000000000000000000000
--- a/assets/solr_service/create_core.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-if [ ! -d /var/solr/data/$1 ];then
- precreate-core $1
- cp managed-schema.xml /var/solr/data/$1/conf/
-fi
diff --git a/assets/solr_service/daily_backup.sh b/assets/solr_service/daily_backup.sh
deleted file mode 100644
index 9df9d8de968d9b89a29f0050abe9c80a826f757a..0000000000000000000000000000000000000000
--- a/assets/solr_service/daily_backup.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-###############################################
-# CREATE a daily backup of the two freva cores
-curl "http://localhost:8983/solr/latest/replication?command=backup&numberToKeep=${NUM_BACKUPS}"
-curl "http://localhost:8983/solr/${CORE}/replication?command=backup&numberToKeep=${NUM_BACKUPS}"
diff --git a/assets/solr_service/managed-schema.xml b/assets/solr_service/managed-schema.xml
deleted file mode 100644
index ca0ae799aec65d8868e1a435a2d1dd4c6d2b819e..0000000000000000000000000000000000000000
--- a/assets/solr_service/managed-schema.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Solr managed schema - automatically generated - DO NOT EDIT -->
-<schema name="file_system" version="1.6">
- <uniqueKey>file</uniqueKey>
- <fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
- <fieldType name="pdates" class="solr.DatePointField" docValues="true" multiValued="true"/>
- <fieldType name="pdoubles" class="solr.DoublePointField" docValues="true" multiValued="true"/>
- <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
- <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
- <fieldType name="plongs" class="solr.LongPointField" docValues="true" multiValued="true"/>
- <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
- <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
-
- <fieldType name="version" class="solr.TextField" >
- <analyzer>
- <charFilter class="solr.PatternReplaceCharFilterFactory" pattern="^v"/>
- <tokenizer class="solr.KeywordTokenizerFactory"/>
- </analyzer>
- </fieldType>
-
- <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
- <analyzer type="index">
- <tokenizer class="solr.KeywordTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- </analyzer>
- <analyzer type="query">
- <tokenizer class="solr.KeywordTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- </analyzer>
- </fieldType>
-
- <field name="_version_" type="plong" indexed="true" stored="true"/>
- <field name="file" type="string" multiValued="false" indexed="true" required="true" stored="true"/>
- <field name="timestamp" type="pfloat" indexed="false" stored="true"/>
- <field name="creation_time" type="pdate" indexed="true" stored="false" default="NOW"/>
-
- <!-- we need this to search for latest version.
- If not it won't work for entries not being versioned at all -->
- <field name="version" type="version" stored="false" indexed="true" default="-1"/>
- <field name="file_no_version" type="string" stored="false" indexed="true"/>
- <field name="_root_" type="string" indexed="false" stored="false" docValues="false"/>
- <dynamicField name="*" type="text_general" stored="true" indexed="true" multiValued="true"/>
-</schema>
diff --git a/assets/vault/Dockerfile b/assets/vault/Dockerfile
index b089059e451cef3f52a730e9c9880a33ccac3503..b895fa3d6637a5a34cde682ef86046e6d64ca6f3 100644
--- a/assets/vault/Dockerfile
+++ b/assets/vault/Dockerfile
@@ -12,13 +12,9 @@ ARG db_name=""
ENV VAULT_ADDR='http://127.0.0.1:8200'
ENV ROOT_PW=${rootpw}
COPY runserver.py /bin/runserver
-COPY vault-server-tls.hcl /vault
-COPY policy-file.hcl /vault
-COPY ${project}.key /mnt/freva.key
-COPY ${project}.crt /mnt/freva.crt
-RUN set -ex ;\
- chown vault:vault /vault/*.hcl;\
- chown vault:vault /mnt/freva.*;\
+COPY --chown=vault:vault vault-server-tls.hcl /vault
+COPY --chown=vault:vault policy-file.hcl /vault
+RUN set -ex &&\
chmod +x /bin/runserver &&\
mkdir -p /data && chown -R vault:vault /data &&\
apk add --update --no-cache python3 mysql mysql-client &&\
diff --git a/assets/vault/runserver.py b/assets/vault/runserver.py
index d3c1c279f39c84939154114dad11f9385e8bfe7f..ea9953f65fb96658dfc72c0cdcdeba471a797f8e 100644
--- a/assets/vault/runserver.py
+++ b/assets/vault/runserver.py
@@ -16,7 +16,7 @@ import requests
app = Flask(__name__)
api = Api(app)
KEY_FILE = Path("/vault/keys")
-CERT_DIR = Path("/mnt")
+CERT_DIR = Path("/data")
def unseal():
@@ -70,10 +70,6 @@ class Vault(Resource):
key = [k.strip() for k in f.readlines() if not k.startswith("-")]
return ("".join(key)).encode()
- @cached_property
- def private_key(self):
- return hashlib.sha512(self._read_key("key")).hexdigest()
-
@cached_property
def public_key(self):
return hashlib.sha512(self._read_key("crt")).hexdigest()
diff --git a/assets/web/create_web_conf.py b/assets/web/create_web_conf.py
deleted file mode 100644
index 0c614694b80b9d150e2ff735bd388af16e5f42d3..0000000000000000000000000000000000000000
--- a/assets/web/create_web_conf.py
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-from pathlib import Path
-
-
-def parse_args(argv=None):
-
- app = argparse.ArgumentParser(
- "apache_config_parsers", description="Create the right apache configuration."
- )
- app.add_argument("input_file", type=Path, help="Input configuration")
- app.add_argument(
- "output_file",
- type=Path,
- help="Target configfile",
- default=Path("/etc/apache2/sites-available"),
- )
- app.add_argument(
- "--website", type=str, default="www.freva.org", help="Set the website"
- )
- app.add_argument("--root-dir", type=Path, help="Set the project root path"),
- app.add_argument("--work-dir", type=Path, help="Set the project root path"),
- app.add_argument("--project-name", type=str, help="Set the name of the project")
- app.add_argument("--alias", type=str, help="Set the server alias")
- app.add_argument("--python", type=str, help="Set the python version")
- app.add_argument("--server-name", type=str, help="Set the name of the host system")
- args = app.parse_args()
- return args
-
-
-def edit_config(
- args,
- project_root="/srv/http",
- variables=(
- "website",
- "root_dir",
- "work_dir",
- "project_name",
- "server_name",
- "python_vers",
- ),
-):
- """Edit the configuration file."""
- with args.input_file.open() as f:
- inp_config = f.read().replace("%PROJECT_ROOT", project_root)
- alias = args.alias
- kwargs = dict(args._get_kwargs())
- kwargs["website"] = "https://" + kwargs["website"].replace("http://", "").replace(
- "https://", ""
- )
- kwargs["python_vers"] = f'python{kwargs["python"]}'
- if alias and alias != "none":
- alias = f'\n{8*" "}'.join(
- [f"ServerAlias {a.strip()}" for a in alias.split(",") if a.strip()]
- )
- inp_config = inp_config.replace("#ServerAlias %SERVER_ALIAS", alias)
- for v in variables:
- inp_config = inp_config.replace(f"%{v.upper()}", str(kwargs[v]))
- with args.output_file.open("w") as f:
- f.write(inp_config)
-
-
-if __name__ == "__main__":
-
- import sys
-
- edit_config(parse_args())
diff --git a/assets/web/freva_web.conf b/assets/web/freva_web.conf
index d06c12e84193c9e5cdca4f096b2e940c629da7f2..e11d4b94cd5b43f3d1ab67a1c0c945574448784f 100644
--- a/assets/web/freva_web.conf
+++ b/assets/web/freva_web.conf
@@ -1,95 +1,401 @@
-LoadModule deflate_module modules/mod_deflate.so
-LoadModule wsgi_module modules/mod_wsgi.so
-LoadModule ssl_module modules/mod_ssl.so
+ServerRoot "/usr/local/apache2"
+Listen 80
+
+#
+# Dynamic Shared Object (DSO) Support
+#
+# To be able to use the functionality of a module which was built as a DSO you
+# have to place corresponding `LoadModule' lines at this location so the
+# directives contained in it are actually available _before_ they are used.
+# Statically compiled modules (those listed by `httpd -l') do not need
+# to be loaded here.
+#
+# Example:
+# LoadModule foo_module modules/mod_foo.so
+#
+LoadModule mpm_event_module modules/mod_mpm_event.so
+#LoadModule mpm_prefork_module modules/mod_mpm_prefork.so
+#LoadModule mpm_worker_module modules/mod_mpm_worker.so
+LoadModule authn_file_module modules/mod_authn_file.so
+#LoadModule authn_dbm_module modules/mod_authn_dbm.so
+#LoadModule authn_anon_module modules/mod_authn_anon.so
+#LoadModule authn_dbd_module modules/mod_authn_dbd.so
+#LoadModule authn_socache_module modules/mod_authn_socache.so
+LoadModule authn_core_module modules/mod_authn_core.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+LoadModule authz_user_module modules/mod_authz_user.so
+#LoadModule authz_dbm_module modules/mod_authz_dbm.so
+#LoadModule authz_owner_module modules/mod_authz_owner.so
+#LoadModule authz_dbd_module modules/mod_authz_dbd.so
+LoadModule authz_core_module modules/mod_authz_core.so
+#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+#LoadModule authnz_fcgi_module modules/mod_authnz_fcgi.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule auth_basic_module modules/mod_auth_basic.so
+#LoadModule auth_form_module modules/mod_auth_form.so
+#LoadModule auth_digest_module modules/mod_auth_digest.so
+#LoadModule allowmethods_module modules/mod_allowmethods.so
+#LoadModule isapi_module modules/mod_isapi.so
+#LoadModule file_cache_module modules/mod_file_cache.so
+#LoadModule cache_module modules/mod_cache.so
+#LoadModule cache_disk_module modules/mod_cache_disk.so
+#LoadModule cache_socache_module modules/mod_cache_socache.so
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
+#LoadModule socache_dbm_module modules/mod_socache_dbm.so
+#LoadModule socache_memcache_module modules/mod_socache_memcache.so
+#LoadModule socache_redis_module modules/mod_socache_redis.so
+#LoadModule watchdog_module modules/mod_watchdog.so
+#LoadModule macro_module modules/mod_macro.so
+#LoadModule dbd_module modules/mod_dbd.so
+#LoadModule bucketeer_module modules/mod_bucketeer.so
+#LoadModule dumpio_module modules/mod_dumpio.so
+#LoadModule echo_module modules/mod_echo.so
+#LoadModule example_hooks_module modules/mod_example_hooks.so
+#LoadModule case_filter_module modules/mod_case_filter.so
+#LoadModule case_filter_in_module modules/mod_case_filter_in.so
+#LoadModule example_ipc_module modules/mod_example_ipc.so
+#LoadModule buffer_module modules/mod_buffer.so
+#LoadModule data_module modules/mod_data.so
+#LoadModule ratelimit_module modules/mod_ratelimit.so
+LoadModule reqtimeout_module modules/mod_reqtimeout.so
+#LoadModule ext_filter_module modules/mod_ext_filter.so
+#LoadModule request_module modules/mod_request.so
+#LoadModule include_module modules/mod_include.so
+LoadModule filter_module modules/mod_filter.so
+#LoadModule reflector_module modules/mod_reflector.so
+#LoadModule substitute_module modules/mod_substitute.so
+#LoadModule sed_module modules/mod_sed.so
+#LoadModule charset_lite_module modules/mod_charset_lite.so
+LoadModule deflate_module modules/mod_deflate.so
+LoadModule xml2enc_module modules/mod_xml2enc.so
+LoadModule proxy_html_module modules/mod_proxy_html.so
+#LoadModule brotli_module modules/mod_brotli.so
+LoadModule mime_module modules/mod_mime.so
+#LoadModule ldap_module modules/mod_ldap.so
+LoadModule log_config_module modules/mod_log_config.so
+#LoadModule log_debug_module modules/mod_log_debug.so
+#LoadModule log_forensic_module modules/mod_log_forensic.so
+#LoadModule logio_module modules/mod_logio.so
+#LoadModule lua_module modules/mod_lua.so
+LoadModule env_module modules/mod_env.so
+#LoadModule mime_magic_module modules/mod_mime_magic.so
+#LoadModule cern_meta_module modules/mod_cern_meta.so
+#LoadModule expires_module modules/mod_expires.so
LoadModule headers_module modules/mod_headers.so
-WSGIApplicationGroup http
-User http
-# WSGISocketPrefix /var/run/wsgi
-ServerName %SERVER_NAME
-<VirtualHost *:80>
- Redirect "/" "%WEBSITE"
-</VirtualHost>
+#LoadModule ident_module modules/mod_ident.so
+#LoadModule usertrack_module modules/mod_usertrack.so
+#LoadModule unique_id_module modules/mod_unique_id.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule version_module modules/mod_version.so
+#LoadModule remoteip_module modules/mod_remoteip.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_connect_module modules/mod_proxy_connect.so
+#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+#LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so
+#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so
+#LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
+#LoadModule proxy_fdpass_module modules/mod_proxy_fdpass.so
+#LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so
+#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so
+#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+#LoadModule proxy_express_module modules/mod_proxy_express.so
+#LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so
+#LoadModule session_module modules/mod_session.so
+#LoadModule session_cookie_module modules/mod_session_cookie.so
+#LoadModule session_crypto_module modules/mod_session_crypto.so
+#LoadModule session_dbd_module modules/mod_session_dbd.so
+#LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
+#LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
+LoadModule ssl_module modules/mod_ssl.so
+#LoadModule optional_hook_export_module modules/mod_optional_hook_export.so
+#LoadModule optional_hook_import_module modules/mod_optional_hook_import.so
+#LoadModule optional_fn_import_module modules/mod_optional_fn_import.so
+#LoadModule optional_fn_export_module modules/mod_optional_fn_export.so
+#LoadModule dialup_module modules/mod_dialup.so
+#LoadModule http2_module modules/mod_http2.so
+#LoadModule proxy_http2_module modules/mod_proxy_http2.so
+#LoadModule md_module modules/mod_md.so
+#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so
+#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so
+#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so
+#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so
+LoadModule unixd_module modules/mod_unixd.so
+#LoadModule heartbeat_module modules/mod_heartbeat.so
+#LoadModule heartmonitor_module modules/mod_heartmonitor.so
+#LoadModule dav_module modules/mod_dav.so
+LoadModule status_module modules/mod_status.so
+LoadModule autoindex_module modules/mod_autoindex.so
+#LoadModule asis_module modules/mod_asis.so
+#LoadModule info_module modules/mod_info.so
+#LoadModule suexec_module modules/mod_suexec.so
+<IfModule !mpm_prefork_module>
+ #LoadModule cgid_module modules/mod_cgid.so
+</IfModule>
+<IfModule mpm_prefork_module>
+ #LoadModule cgi_module modules/mod_cgi.so
+</IfModule>
+#LoadModule dav_fs_module modules/mod_dav_fs.so
+#LoadModule dav_lock_module modules/mod_dav_lock.so
+#LoadModule vhost_alias_module modules/mod_vhost_alias.so
+#LoadModule negotiation_module modules/mod_negotiation.so
+LoadModule dir_module modules/mod_dir.so
+#LoadModule imagemap_module modules/mod_imagemap.so
+#LoadModule actions_module modules/mod_actions.so
+#LoadModule speling_module modules/mod_speling.so
+#LoadModule userdir_module modules/mod_userdir.so
+LoadModule alias_module modules/mod_alias.so
+LoadModule rewrite_module modules/mod_rewrite.so
+
+<IfModule unixd_module>
+#
+# If you wish httpd to run as a different user or group, you must run
+# httpd as root initially and it will switch.
+#
+# User/Group: The name (or #number) of the user/group to run httpd as.
+# It is usually good practice to create a dedicated user and group for
+# running httpd, as with most system services.
+#
+User www-data
+Group www-data
+
+</IfModule>
+
+
+ServerName www-regiklim.dkrz.de
+ServerAdmin you@example.com
+
+ServerSignature Off
+ServerTokens Prod
+MaxKeepAliveRequests 100
+Timeout 60
+KeepAliveTimeout 5
+
+RewriteEngine On
+RewriteCond %{HTTPS} off
+RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI}
+
Listen 443
+
+#SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog
+SSLSessionCache shmcb:/run/httpd/sslcache(512000)
+SSLSessionCacheTimeout 300
+SSLRandomSeed startup file:/dev/urandom 256
+SSLRandomSeed connect builtin
+SSLCryptoDevice builtin
+
<VirtualHost *:443>
- # The ServerName directive sets the request scheme, hostname and port that
- # the server uses to identify itself. This is used when creating
- # redirection URLs. In the context of virtual hosts, the ServerName
- # specifies what hostname must appear in the request's Host: header to
- # match this virtual host. For the default virtual host (this file) this
- # value is not decisive as it is used as a last resort host regardless.
- # However, you must set it for any further virtual host explicitly.
- # ServerName %SERVER_NAME
- ServerAdmin webmaster@localhost
- DocumentRoot %PROJECT_ROOT
- #ServerAlias %SERVER_ALIAS
- ##Use separate log files for the SSL virtual host;
- LogLevel debug
- Alias /robots.txt %PROJECT_ROOT/static/robots.txt
- Alias /favicon.ico %PROJECT_ROOT/static/favicon.ico
- AliasMatch ^/([^/]*\.css) %PROJECT_ROOT/static/styles/$1
- Alias /media %PROJECT_ROOT/media
- Alias /static %PROJECT_ROOT/static
- SSLEngine on
- SSLCertificateFile %ROOT_DIR/freva/%PROJECT_NAME.crt
- SSLCertificateKeyFile %ROOT_DIR/freva/%PROJECT_NAME.key
- Header always add Strict-Transport-Security "max-age=15768000"
- Header set X-Frame-Options "SAMEORIGIN"
- Header set X-XSS-Protecton "1; mode=block"
- Header set X-Content-Type-Options "nosniff"
- Header set Referrer-Policy "same-origin"
- <Directory %ROOT_DIR/*>
- Options Indexes MultiViews
- AllowOverride None
- Require all granted
- </Directory>
- <Directory %WORK_DIR/*>
- Options Indexes MultiViews
- AllowOverride None
- Require all granted
- </Directory>
- <Directory %PROJECT_ROOT/static>
- Options Indexes MultiViews
- AllowOverride None
- Require all granted
- AddOutputFilterByType DEFLATE application/json
- </Directory>
- <Directory %PROJECT_ROOT/doc>
- Options FollowSymLinks
- AllowOverride None
- Require all granted
- </Directory>
- <Directory %PROJECT_ROOT/media>
- Options Indexes MultiViews
- AllowOverride None
- Require all granted
- </Directory>
- <Directory %PROJECT_ROOT/static>
- Require all granted
- </Directory>
- <Directory /tmp/tail/*>
- Options Indexes MultiViews
- AllowOverride None
- Require all granted
- </Directory>
- WSGIProcessGroup freva_web
- WSGIApplicationGroup %{GLOBAL}
- WSGIDaemonProcess freva_web display-name=%{GROUP} python-path=%PROJECT_ROOT
- WSGIScriptAlias / %PROJECT_ROOT/django_evaluation/wsgi.py
- <Directory %PROJECT_ROOT/django_evaluation>
- <Files wsgi.py>
- Require all granted
- </Files>
- </Directory>
- # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
- # error, crit, alert, emerg.
- # It is also possible to configure the loglevel for particular
- # modules, e.g.
- LogLevel info ssl:warn
- # For most configuration files from conf-available/, which are
- # enabled or disabled at a global level, it is possible to
- # include a line for only one particular virtual host. For example the
- # following line enables the CGI configuration for this host only
- # after it has been globally disabled with "a2disconf".
- #Include conf-available/serve-cgi-bin.conf
+ ProxyPass / http://${FREVA_HOST}:8000/
+ ProxyPassReverse / http://${FREVA_HOST}:8000/
+ SSLEngine on
+ SSLProtocol all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1
+ Header set X-Frame-Options "SAMEORIGIN"
+
+ SSLCipherSuite ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256
+ SSLHonorCipherOrder on
+
+ SSLCertificateFile /etc/ssl/certs/server-cert.crt
+ SSLCertificateKeyFile /etc/ssl/private/server-key.key
+ SSLCertificateChainFile /etc/ssl/certs/cacert.pem
+ SetOutputFilter DEFLATE
+ SetEnvIfNoCase Request_URI "\.(?:gif|jpe?g|png)$" no-gzip
+ Header always set Strict-Transport-Security "max-age=15552000; includeSubdomains;"
</VirtualHost>
+
+<Files ".ht*">
+ Require all denied
+</Files>
+
+
+ErrorLog /proc/self/fd/2
+
+
+LogLevel info
+
+<IfModule log_config_module>
+ #
+ # The following directives define some format nicknames for use with
+ # a CustomLog directive (see below).
+ #
+ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+ LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+ <IfModule logio_module>
+ # You need to enable mod_logio.c to use %I and %O
+ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+ </IfModule>
+
+ #
+ # The location and format of the access logfile (Common Logfile Format).
+ # If you do not define any access logfiles within a <VirtualHost>
+ # container, they will be logged here. Contrariwise, if you *do*
+ # define per-<VirtualHost> access logfiles, transactions will be
+ # logged therein and *not* in this file.
+ #
+ # CustomLog /proc/self/fd/1 common
+ CustomLog "|/usr/bin/tee /usr/local/apache2/logs/access.log" common
+
+ #
+ # If you prefer a logfile with access, agent, and referer information
+ # (Combined Logfile Format) you can use the following directive.
+ #
+ #CustomLog "logs/access_log" combined
+</IfModule>
+
+
+<IfModule cgid_module>
+ #
+ # ScriptSock: On threaded servers, designate the path to the UNIX
+ # socket used to communicate with the CGI daemon of mod_cgid.
+ #
+ #Scriptsock cgisock
+</IfModule>
+
+<IfModule headers_module>
+ #
+ # Avoid passing HTTP_PROXY environment to CGI's on this or any proxied
+ # backend servers which have lingering "httpoxy" defects.
+ # 'Proxy' request header is undefined by the IETF, not listed by IANA
+ #
+ RequestHeader unset Proxy early
+</IfModule>
+
+<IfModule mime_module>
+ #
+ # TypesConfig points to the file containing the list of mappings from
+ # filename extension to MIME-type.
+ #
+ TypesConfig conf/mime.types
+
+ #
+ # AddType allows you to add to or override the MIME configuration
+ # file specified in TypesConfig for specific file types.
+ #
+ #AddType application/x-gzip .tgz
+ #
+ # AddEncoding allows you to have certain browsers uncompress
+ # information on the fly. Note: Not all browsers support this.
+ #
+ #AddEncoding x-compress .Z
+ #AddEncoding x-gzip .gz .tgz
+ #
+ # If the AddEncoding directives above are commented-out, then you
+ # probably should define those extensions to indicate media types:
+ #
+ AddType application/x-compress .Z
+ AddType application/x-gzip .gz .tgz
+
+ #
+ # AddHandler allows you to map certain file extensions to "handlers":
+ # actions unrelated to filetype. These can be either built into the server
+ # or added with the Action directive (see below)
+ #
+ # To use CGI scripts outside of ScriptAliased directories:
+ # (You will also need to add "ExecCGI" to the "Options" directive.)
+ #
+ #AddHandler cgi-script .cgi
+
+ # For type maps (negotiated resources):
+ #AddHandler type-map var
+
+ #
+ # Filters allow you to process content before it is sent to the client.
+ #
+ # To parse .shtml files for server-side includes (SSI):
+ # (You will also need to add "Includes" to the "Options" directive.)
+ #
+ #AddType text/html .shtml
+ #AddOutputFilter INCLUDES .shtml
+</IfModule>
+
+#
+# The mod_mime_magic module allows the server to use various hints from the
+# contents of the file itself to determine its type. The MIMEMagicFile
+# directive tells the module where the hint definitions are located.
+#
+#MIMEMagicFile conf/magic
+
+#
+# Customizable error responses come in three flavors:
+# 1) plain text 2) local redirects 3) external redirects
+#
+# Some examples:
+#ErrorDocument 500 "The server made a boo boo."
+#ErrorDocument 404 /missing.html
+#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
+#ErrorDocument 402 http://www.example.com/subscription_info.html
+#
+
+#
+# MaxRanges: Maximum number of Ranges in a request before
+# returning the entire resource, or one of the special
+# values 'default', 'none' or 'unlimited'.
+# Default setting is to accept 200 Ranges.
+#MaxRanges unlimited
+
+#
+# EnableMMAP and EnableSendfile: On systems that support it,
+# memory-mapping or the sendfile syscall may be used to deliver
+# files. This usually improves server performance, but must
+# be turned off when serving from networked-mounted
+# filesystems or if support for these functions is otherwise
+# broken on your system.
+# Defaults: EnableMMAP On, EnableSendfile Off
+#
+#EnableMMAP off
+#EnableSendfile on
+
+# Supplemental configuration
+#
+# The configuration files in the conf/extra/ directory can be
+# included to add extra features or to modify the default configuration of
+# the server, or you may simply copy their contents here and change as
+# necessary.
+
+# Server-pool management (MPM specific)
+#Include conf/extra/httpd-mpm.conf
+
+# Multi-language error messages
+#Include conf/extra/httpd-multilang-errordoc.conf
+
+# Fancy directory listings
+#Include conf/extra/httpd-autoindex.conf
+
+# Language settings
+#Include conf/extra/httpd-languages.conf
+
+# User home directories
+#Include conf/extra/httpd-userdir.conf
+
+# Real-time info on requests and configuration
+#Include conf/extra/httpd-info.conf
+
+# Virtual hosts
+#Include conf/extra/httpd-vhosts.conf
+
+# Local access to the Apache HTTP Server Manual
+#Include conf/extra/httpd-manual.conf
+
+# Distributed authoring and versioning (WebDAV)
+#Include conf/extra/httpd-dav.conf
+
+# Various default settings
+#Include conf/extra/httpd-default.conf
+
+# Configure mod_proxy_html to understand HTML4/XHTML1
+#<IfModule proxy_html_module>
+#Include conf/extra/proxy-html.conf
+#</IfModule>
+
+# Secure (SSL/TLS) connections
+#Include conf/extra/httpd-ssl.conf
+#
+# Note: The following must must be present to support
+# starting without SSL on platforms with no /dev/random equivalent
+# but a statically compiled-in mod_ssl.
+#
+<IfModule ssl_module>
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+</IfModule>
diff --git a/assets/web/setup_web.sh b/assets/web/setup_web.sh
index cf8b84d2a9531fe09abf0c18b2e374487827c4fc..fff6df3f38c2f8fd4412d38d3f3794c17230851a 100644
--- a/assets/web/setup_web.sh
+++ b/assets/web/setup_web.sh
@@ -6,21 +6,9 @@ if [ -z $1 ];then
exit 1
fi
-web_dir=$(mktemp -u)
-git clone -b $GIT_BRANCH $GIT_URL $web_dir
-cd $PROJECT_ROOT
-rm -fr .git
-cp -rT ${web_dir} $PROJECT_ROOT
-rm -r ${web_dir}
-mkdir -p ${PROJECT_ROOT}/static/preview
-cp /etc/freva_web.conf .
python manage.py migrate --fake contenttypes
python manage.py migrate --fake-initial --noinput
python manage.py createsuperuser \
--noinput \
--username freva-admin \
- --email $DJANGO_SUPERUSER_EMAIL
-python manage.py collectstatic --noinput
-npm install
-npm run build-production
-python manage.py collectstatic --noinput
+ --email $DJANGO_SUPERUSER_EMAIL || echo 0
diff --git a/src/freva_deployment/__init__.py b/src/freva_deployment/__init__.py
index f382a135a146816edade43fadcd6cf46a427687f..9a112f25e8dee30360b35f524ac747ae7be35b03 100644
--- a/src/freva_deployment/__init__.py
+++ b/src/freva_deployment/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2205.0.15"
+__version__ = "2205.1.0"
AVAILABLE_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
AVAILABLE_CONDA_ARCHS = [
"Linux-x86_64",
diff --git a/src/freva_deployment/cli/_deploy.py b/src/freva_deployment/cli/_deploy.py
index e087cea0c519f2c921e77adbd6af08e521e9cc9a..4d58954889dc45adad0438cacc7b0d30c95c5a10 100644
--- a/src/freva_deployment/cli/_deploy.py
+++ b/src/freva_deployment/cli/_deploy.py
@@ -38,8 +38,8 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
"--steps",
type=str,
nargs="+",
- default=["services", "web", "core"],
- choices=["services", "web", "core", "db", "solr", "backup"],
+ default=["db", "solr", "web", "core"],
+ choices=["web", "core", "db", "solr"],
help="The services/code stack to be deployed",
)
ap.add_argument(
@@ -57,17 +57,7 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
action="version",
version="%(prog)s {version}".format(version=__version__),
)
- args = ap.parse_args()
- services = {"services": ["db", "vault", "solr", "backup"]}
- steps = []
-
- for step in args.steps:
- try:
- steps += services[step]
- except KeyError:
- steps += [step]
- args.steps = steps
- return args
+ return ap.parse_args()
def cli(argv: list[str] | None = None) -> None:
diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py
index 6cea58b9f755a4957f14ca2c709d2f89c2d96180..de50a532f3084d60fcde361f3295e685a716d0e9 100644
--- a/src/freva_deployment/deploy.py
+++ b/src/freva_deployment/deploy.py
@@ -8,6 +8,7 @@ import shlex
import string
from subprocess import run
import sys
+from urllib.parse import urlparse
from tempfile import TemporaryDirectory, mkdtemp
from typing import Any
@@ -18,7 +19,6 @@ import yaml
from .utils import (
asset_dir,
config_dir,
- create_self_signed_cert,
get_passwd,
logger,
upload_server_map,
@@ -35,8 +35,6 @@ class DeployFactory:
The name of the project to distinguish this instance from others.
steps: list[str], default: ["services", "core", "web"]
The components that are going to be deployed.
- cert_file: os.PathLike, default: None
- The path to the public cert file that is injected to the web container.
config_file: os.PathLike, default: None
Path to any existing deployment configuration file.
@@ -68,28 +66,33 @@ class DeployFactory:
self._inv_tmpl = Path(config_file or config_dir / "inventory.toml")
self._cfg_tmpl = self.aux_dir / "evaluation_system.conf.tmpl"
self.cfg = self._read_cfg()
- self._cert_file = self.cfg.pop("cert_file", "")
self.project_name = self.cfg.pop("project_name", None)
if not self.project_name:
raise ValueError("You must set a project name")
+ @property
+ def public_key_file(self) -> str:
+ """Path to the public certificate file."""
+ public_keyfile = self.cfg["certificates"].get("public_keyfile")
+ if public_keyfile:
+ return str(Path(public_keyfile).expanduser().absolute())
+ raise ValueError("You must give a valid path to a public key file.")
+
@property
def private_key_file(self) -> str:
- """Path to the private certifcate file."""
- return str(Path(self.public_key_file).with_suffix(".key"))
+ """Path to the private key file."""
+ keyfile = self.cfg["certificates"].get("private_keyfile")
+ if keyfile:
+ return str(Path(keyfile).expanduser().absolute())
+ raise ValueError("You must give a valid path to a private key file.")
@property
- def public_key_file(self) -> str:
- """Path to the public certificate file."""
- if not any((step in self._steps_with_cert for step in self.steps)):
- return str(Path(mkdtemp(suffix=".crt")))
- if not self._cert_file:
- _cert_file = config_dir / "keys" / f"{self.project_name}.crt"
- _cert_file = Path(self._cert_file or _cert_file)
- if not _cert_file.is_file():
- logger.warning("Certificate file does not exist, creating new one")
- _cert_file = create_self_signed_cert(_cert_file)
- return str(_cert_file.absolute())
+ def chain_key_file(self) -> str:
+ """Path to the private key file."""
+ keyfile = self.cfg["certificates"].get("chain_keyfile")
+ if keyfile:
+ return str(Path(keyfile).expanduser().absolute())
+ raise ValueError("You must give a valid path to a chain key file.")
def _prep_vault(self) -> None:
"""Prepare the vault."""
@@ -100,7 +103,6 @@ class DeployFactory:
self.cfg["vault"]["config"]["root_passwd"] = self.master_pass
self.cfg["vault"]["config"]["passwd"] = self.db_pass
self.cfg["vault"]["config"]["keyfile"] = self.public_key_file
- self.cfg["vault"]["config"]["private_keyfile"] = self.private_key_file
self.cfg["vault"]["config"]["email"] = ",".join(
self.cfg["web"]["config"].get("contacts", [])
)
@@ -114,7 +116,6 @@ class DeployFactory:
self.cfg["db"]["config"]["root_passwd"] = self.master_pass
self.cfg["db"]["config"]["passwd"] = self.db_pass
self.cfg["db"]["config"]["keyfile"] = self.public_key_file
- self.cfg["db"]["config"]["private_keyfile"] = self.private_key_file
for key in ("name", "user", "db"):
self.cfg["db"]["config"][key] = self.cfg["db"]["config"].get(key) or "freva"
db_host = self.cfg["db"]["config"].get("host", "")
@@ -125,7 +126,6 @@ class DeployFactory:
self.cfg["web"]["config"].get("contacts", [])
)
self._prep_vault()
- self._create_sql_dump()
def _prep_solr(self) -> None:
"""prepare the apache solr service."""
@@ -147,24 +147,20 @@ class DeployFactory:
)
if not self.cfg["core"]["config"]["admins"]:
self.cfg["core"]["config"]["admins"] = getuser()
- self.cfg["core"]["config"]["branch"] = (
- self.cfg["core"]["config"].get("branch") or "freva-dev"
- )
install_dir = self.cfg["core"]["config"]["install_dir"]
root_dir = self.cfg["core"]["config"].get("root_dir", "")
if not root_dir:
self.cfg["core"]["config"]["root_dir"] = install_dir
self.cfg["core"]["config"]["keyfile"] = self.public_key_file
- self.cfg["core"]["config"]["private_keyfile"] = self.private_key_file
git_exe = self.cfg["core"]["config"].get("git_path")
self.cfg["core"]["config"]["git_path"] = git_exe or "git"
+ self.cfg["core"]["config"][
+ "git_url"
+ ] = "https://gitlab.dkrz.de/freva/evaluation_system.git"
def _prep_web(self) -> None:
"""prepare the web deployment."""
self._config_keys.append("web")
- self.cfg["web"]["config"]["branch"] = (
- self.cfg["web"]["config"].get("branch") or "main"
- )
self._prep_core()
admin = self.cfg["core"]["config"]["admins"]
if not isinstance(admin, str):
@@ -188,19 +184,27 @@ class DeployFactory:
_webserver_items["homepage_text"] = f_obj.read()
except (FileNotFoundError, IOError, KeyError):
pass
- logo = Path(self.cfg["web"]["config"].get("institution_logo", ""))
- alias = self.cfg["web"]["config"].pop("server_alias", [])
- if isinstance(alias, str):
- alias = alias.split(",")
- alias = ",".join([a for a in alias if a.strip()])
- if not alias:
- alias = self.cfg["web"]["hosts"]
- self.cfg["web"]["config"]["server_alias"] = alias
+ server_name = self.cfg["web"]["config"].pop("server_name", [])
+ if isinstance(server_name, str):
+ server_name = server_name.split(",")
+ server_name = ",".join([a for a in server_name if a.strip()])
+ if not server_name:
+ server_name = self.cfg["web"]["hosts"]
+ self.cfg["web"]["config"]["server_name"] = server_name
web_host = self.cfg["web"]["hosts"]
if web_host == "127.0.0.1":
web_host = "localhost"
self.cfg["web"]["config"]["host"] = web_host
- _webserver_items["INSTITUTION_LOGO"] = f"logo{logo.suffix}"
+ _webserver_items["INSTITUTION_LOGO"] = "/path/to/your/logo"
+ trusted_origin = urlparse(server_name)
+ if trusted_origin.scheme:
+ _webserver_items["CSRF_TRUSTED_ORIGINS"] = [
+ f"https://{urlparse(server_name).netloc}"
+ ]
+ else:
+ _webserver_items["CSRF_TRUSTED_ORIGINS"] = [
+ f"https://{urlparse(server_name).path}"
+ ]
_webserver_items["FREVA_BIN"] = os.path.join(
self.cfg["core"]["config"]["install_dir"], "bin"
)
@@ -216,13 +220,13 @@ class DeployFactory:
with self.web_conf_file.open("w") as f_obj:
toml.dump(_webserver_items, f_obj)
for key in ("core", "web"):
- logo_suffix = logo.suffix or ".png"
self.cfg[key]["config"]["config_toml_file"] = str(self.web_conf_file)
- self.cfg[key]["config"]["institution_logo"] = str(logo.absolute())
- self.cfg[key]["config"]["institution_logo_suffix"] = logo_suffix
if not self.master_pass:
self.master_pass = get_passwd()
self.cfg["web"]["config"]["root_passwd"] = self.master_pass
+ self.cfg["web"]["config"]["private_keyfile"] = self.private_key_file
+ self.cfg["web"]["config"]["public_keyfile"] = self.public_key_file
+ self.cfg["web"]["config"]["chain_keyfile"] = self.chain_key_file
def __enter__(self):
return self
@@ -237,46 +241,6 @@ class DeployFactory:
except FileNotFoundError as error:
raise FileNotFoundError(f"No such file {self._inv_tmpl}") from error
- @property
- def git_url(self) -> str:
- """Get the url of the git repository."""
- try:
- return self.cfg["coreservers:vars"]["git_url"]
- except KeyError as error:
- raise KeyError(
- (
- "You must set git_url and branch keys in coreservers:vars in "
- "the inventory file"
- )
- ) from error
-
- @property
- def git_branch(self) -> str:
- """Get the branch that is installed."""
- try:
- return self.cfg["coreservers:vars"]["branch"]
- except KeyError:
- return "main"
-
- def _add_local_config(self, config):
- if "db" not in self.steps:
- return {}
- cfg = dict(
- local=dict(
- hosts="127.0.0.1",
- vars=dict(
- db_db=config["db_db"],
- db_passwd=self.db_pass,
- db_host=config["db_host"],
- db_dump=str(self._dump_file),
- db_port=config.get("db_port", 3306),
- user=getuser(),
- local_ansible_python_interpreter=str(self.python_prefix),
- ),
- )
- )
- return cfg
-
def _check_config(self) -> None:
sections = []
for section in self.cfg.keys():
@@ -298,8 +262,6 @@ class DeployFactory:
def _get_files_copy(self, key) -> Path | None:
return dict(
- db=self._dump_file,
- solr=(self.aux_dir / "managed-schema.xml").absolute(),
core=self.eval_conf_file.absolute(),
web=self.eval_conf_file.absolute(),
).get(key, None)
@@ -318,7 +280,11 @@ class DeployFactory:
"".join([random.choice(punctuations) for i in range(num_punctuations)]),
]
str_characters = "".join(characters)
- self._db_pass = "".join(random.sample(str_characters, len(str_characters)))
+ _db_pass = "".join(random.sample(str_characters, len(str_characters)))
+ while _db_pass.startswith("@"):
+ # Vault treats values starting with "@" as file names.
+ _db_pass = "".join(random.sample(str_characters, len(str_characters)))
+ self._db_pass = _db_pass
return self._db_pass
@property
@@ -331,7 +297,7 @@ class DeployFactory:
) -> None:
"""Set additional values to the configuration."""
if step in self._needs_core:
- for key in ("git_url", "branch", "root_dir", "base_dir_location"):
+ for key in ("root_dir", "base_dir_location"):
value = self.cfg["core"]["config"][key]
config[step]["vars"][f"core_{key}"] = value
config[step]["vars"][f"{step}_hostname"] = self.cfg[step]["hosts"]
@@ -366,41 +332,8 @@ class DeployFactory:
config[step]["vars"]["project_name"] = self.project_name
# Add additional keys
self._set_additional_config_values(step, config)
- if "db" in self._config_keys:
- config.update(self._add_local_config(config["db"]["vars"]))
return yaml.dump(config)
- def _create_sql_dump(self):
- """Create a sql dump file to create tables."""
-
- logger.info("Creating database dump file.")
- head = """
-FLUSH PRIVILEGES;
-USE mysql;
-CREATE USER IF NOT EXISTS '{user}'@'localhost' IDENTIFIED BY '{passwd}';
-CREATE USER IF NOT EXISTS '{user}'@'%' IDENTIFIED BY '{passwd}';
-CREATE DATABASE IF NOT EXISTS {db};
-ALTER USER '{user}'@'localhost' IDENTIFIED BY '{passwd}';
-ALTER USER '{user}'@'%' IDENTIFIED BY '{passwd}';
-FLUSH PRIVILEGES;
-GRANT ALL PRIVILEGES ON {db}.* TO '{user}'@'%' WITH GRANT OPTION;
-GRANT ALL PRIVILEGES ON {db}.* TO '{user}'@'localhost' WITH GRANT OPTION;
-FLUSH PRIVILEGES;
-USE {db};
-
-""".format(
- user=self.cfg["db"]["config"]["user"],
- db=self.cfg["db"]["config"]["db"],
- passwd=self.db_pass,
- )
- with self._dump_file.open("w") as f_obj:
- tail = (self.aux_dir / "create_tables.sql").open("r").read()
- f_obj.write(head + tail)
-
- @property
- def _dump_file(self) -> Path:
- return Path(self._td.name) / "sql_dump.sql"
-
@property
def _playbook_file(self) -> Path:
@@ -503,6 +436,11 @@ USE {db};
inventory_str = inventory
RichConsole.print(inventory_str, style="bold", markup=True)
logger.info("Playing the playbooks with ansible")
+ RichConsole.print(
+ "[b]Note:[/] The [blue]BECOME[/] password refers to the "
+ "[blue]sudo[/] password",
+ markup=True,
+ )
v_string = sign(verbosity) * "-" + verbosity * "v"
cmd = (
f"ansible-playbook {v_string} -i {self.inventory_file} "
diff --git a/src/freva_deployment/ui/deployment_tui/__init__.py b/src/freva_deployment/ui/deployment_tui/__init__.py
index 4f4b133e96f65757de7ddf04c2467b9f650cd57e..ce6f5218138871df40e0526fe1de211dece52464 100644
--- a/src/freva_deployment/ui/deployment_tui/__init__.py
+++ b/src/freva_deployment/ui/deployment_tui/__init__.py
@@ -46,5 +46,6 @@ def tui() -> None:
if setup:
server_map = setup.pop("server_map")
ask_pass = setup.pop("ask_pass")
+ print(setup)
with DeployFactory(**setup) as DF:
DF.play(server_map, ask_pass, verbosity)
diff --git a/src/freva_deployment/ui/deployment_tui/base.py b/src/freva_deployment/ui/deployment_tui/base.py
index b10b2735fb601c5f7127615b29cad2ecfa5b1981..590a1e6028a5d15546c8f4882c04468cb563ce7e 100644
--- a/src/freva_deployment/ui/deployment_tui/base.py
+++ b/src/freva_deployment/ui/deployment_tui/base.py
@@ -94,6 +94,14 @@ class BaseForm(npyscreen.FormMultiPageWithMenus, npyscreen.FormWithMenus):
"""Base class for forms."""
_num: int = 0
+ input_fields: dict[str, tuple[npyscreen.TitleText, bool]] = {}
+ """Dictionary of input fileds: the key of the dictionary represents the name
+ of the key in the in config toml input files. Values represent a tuple of
+ npysceen types that display the input information on this key to the
+ user and a boolean indicating whether or not this variable is mandatory.
+ """
+ certificates: list[str] = []
+ """The type of certificate files this step needs."""
def get_config(self, key) -> dict[str, str | bool | list[str]]:
"""Read the configuration for a step."""
diff --git a/src/freva_deployment/ui/deployment_tui/deploy_forms.py b/src/freva_deployment/ui/deployment_tui/deploy_forms.py
index e214743ad4d39e57185b61a9449fdac11a2d8ff9..f0b0c8a66eb392bbb9fd9549960f1280690dbd50 100644
--- a/src/freva_deployment/ui/deployment_tui/deploy_forms.py
+++ b/src/freva_deployment/ui/deployment_tui/deploy_forms.py
@@ -35,6 +35,9 @@ class CoreScreen(BaseForm):
"""Form for the core deployment configuration."""
step: str = "core"
+ """Name of this step."""
+ certificates: list[str] = ["public"]
+ """The type of certificate files this step needs."""
def _add_widgets(self) -> None:
"""Add widgets to the screen."""
@@ -51,14 +54,6 @@ class CoreScreen(BaseForm):
),
True,
),
- branch=(
- self.add_widget_intelligent(
- npyscreen.TitleText,
- name=f"{self.num}Deploy branch:",
- value=cfg.get("branch", "freva-dev"),
- ),
- True,
- ),
install_dir=(
self.add_widget_intelligent(
npyscreen.TitleFilename,
@@ -182,6 +177,8 @@ class WebScreen(BaseForm):
"""Form for the web deployment configuration."""
step: str = "web"
+ certificates: list[str] = ["public", "private", "chain"]
+ """The type of certificate files this step needs."""
def _add_widgets(self) -> None:
"""Add widgets to the screen."""
@@ -201,25 +198,6 @@ class WebScreen(BaseForm):
),
True,
),
- wipe=(
- self.add_widget_intelligent(
- npyscreen.RoundCheckBox,
- max_height=2,
- value=cfg.get("wipe", False),
- editable=True,
- name=(f"{self.num}Delete existing data?"),
- scroll_exit=True,
- ),
- True,
- ),
- branch=(
- self.add_widget_intelligent(
- npyscreen.TitleText,
- name=f"{self.num}Deploy branch:",
- value=cfg.get("branch", "main"),
- ),
- True,
- ),
project_website=(
self.add_widget_intelligent(
npyscreen.TitleText,
@@ -228,14 +206,6 @@ class WebScreen(BaseForm):
),
True,
),
- institution_logo=(
- self.add_widget_intelligent(
- npyscreen.TitleText,
- name=f"{self.num}Path to the logo - leave blank for default logo:",
- value=cfg.get("institution_logo", ""),
- ),
- False,
- ),
main_color=(
self.add_widget_intelligent(
npyscreen.TitleText,
@@ -320,11 +290,11 @@ class WebScreen(BaseForm):
),
True,
),
- home_page_heading=(
+ homepage_heading=(
self.add_widget_intelligent(
npyscreen.TitleText,
name=f"{self.num}A brief describtion of the project:",
- value=cfg.get("home_page_heading", "Lorem ipsum dolor sit amet"),
+ value=cfg.get("homepage_heading", "Lorem ipsum dolor sit amet"),
),
True,
),
@@ -347,7 +317,7 @@ class WebScreen(BaseForm):
),
value=cfg.get(
"auth_ldap_server_uri",
- "ldap://mldap0.hpc.dkrz.de, ldap://mldap1.hpc.dkrz.de",
+ "ldap://idm-dmz.dkrz.de",
),
),
True,
@@ -610,24 +580,32 @@ class RunForm(npyscreen.FormMultiPageAction):
if missing_form:
self.parentApp.change_form(missing_form)
return
- cert_file: str = self.cert_file.value or ""
- if cert_file:
- if not Path(cert_file).exists() or not Path(cert_file).is_file():
- is_file = Path(cert_file).exists()
- msg = f"Public certificate file `{cert_file}` {is_file} must exist or empty."
- npyscreen.notify_confirm(msg, title="ERROR")
- return
- if cert_file:
- cert_file = str(Path(cert_file).expanduser().absolute())
- self.cert_file.value = cert_file
+ cert_files = dict(
+ public=self.public_keyfile.value or "",
+ private=self.private_keyfile.value or "",
+ chain=self.chain_keyfile.value or "",
+ )
+ for key_type, keyfile in cert_files.items():
+ for step, deploy_form in self.parentApp._forms.items():
+ if not keyfile or not Path(keyfile).is_file():
+ if (
+ key_type in deploy_form.certificates
+ and step in self.parentApp.steps
+ ):
+ if keyfile:
+ msg = f"{key_type} certificate file `{keyfile}` must exist."
+ else:
+ msg = f"You must give a {key_type} certificate file."
+ npyscreen.notify_confirm(msg, title="ERROR")
+ return
save_file = self.parentApp.save_config_to_file(write_toml_file=True)
if isinstance(save_file, Path):
save_file = str(save_file)
self.parentApp.setup = {
"server_map": self.server_map.value,
"steps": list(set(self.parentApp.steps)),
- "config_file": save_file or None,
"ask_pass": bool(self.use_ssh_pw.value),
+ "config_file": save_file or None,
}
self.parentApp.exit_application(msg="Do you want to continue?")
@@ -667,10 +645,20 @@ class RunForm(npyscreen.FormMultiPageAction):
name=(f"{self.num}Hostname of the service mapping the freva server arch."),
value=self.parentApp._read_cache("server_map", ""),
)
- self.cert_file = self.add_widget_intelligent(
+ self.public_keyfile = self.add_widget_intelligent(
+ npyscreen.TitleFilename,
+ name=f"{self.num}Select a public certificate file; needed for steps web, core, db",
+ value=self.parentApp.read_cert_file("public_keyfile"),
+ )
+ self.private_keyfile = self.add_widget_intelligent(
+ npyscreen.TitleFilename,
+ name=f"{self.num}Select a private certificate file; needed for steps web",
+ value=self.parentApp.read_cert_file("private_keyfile"),
+ )
+ self.chain_keyfile = self.add_widget_intelligent(
npyscreen.TitleFilename,
- name=f"{self.num}Select a public certificate file, defaults to `<project_name>.crt`",
- value=str(self.parentApp.cert_file or ""),
+ name=f"{self.num}Select a chain certificate file; needed for steps web",
+ value=self.parentApp.read_cert_file("chain_keyfile"),
)
self.use_ssh_pw = self.add_widget_intelligent(
npyscreen.RoundCheckBox,
diff --git a/src/freva_deployment/ui/deployment_tui/main_window.py b/src/freva_deployment/ui/deployment_tui/main_window.py
index fa98f04420d20f7aae896fc6432f6b4328cb30a0..7065fc1151870f4698bf09f14717825f3aea589f 100644
--- a/src/freva_deployment/ui/deployment_tui/main_window.py
+++ b/src/freva_deployment/ui/deployment_tui/main_window.py
@@ -150,7 +150,7 @@ class MainApp(npyscreen.NPSAppManaged):
except Exception as error:
npyscreen.notify_confirm(
title="Error",
- message=f"Couldn't save config:\n{error.__str__()}",
+ message=f"Couldn't save config:\n{error}",
)
return None
@@ -161,11 +161,14 @@ class MainApp(npyscreen.NPSAppManaged):
save_file = str(Path(save_file).expanduser().absolute())
else:
save_file = None
- cert_file = self._setup_form.cert_file.value
- if cert_file:
- cert_file = str(Path(cert_file).expanduser().absolute())
- else:
- cert_file = ""
+ cert_files = dict(
+ public_keyfile=self._setup_form.public_keyfile.value or "",
+ private_keyfile=self._setup_form.private_keyfile.value or "",
+ chain_keyfile=self._setup_form.chain_keyfile.value or "",
+ )
+ for key, value in cert_files.items():
+ if value:
+ cert_files[key] = str(Path(value).expanduser().absolute())
project_name = self._setup_form.project_name.value
server_map = self._setup_form.server_map.value
ssh_pw = self._setup_form.use_ssh_pw.value
@@ -177,9 +180,9 @@ class MainApp(npyscreen.NPSAppManaged):
"project_name": project_name,
"ssh_pw": ssh_pw,
"server_map": server_map,
- "cert_file": cert_file,
"config": self.config,
}
+ config.update(cert_files)
with open(self.cache_dir / "freva_deployment.json", "w") as f:
json.dump({k: v for (k, v) in config.items()}, f, indent=3)
if write_toml_file is False:
@@ -190,13 +193,15 @@ class MainApp(npyscreen.NPSAppManaged):
config_tmpl = cast(Dict[str, Any], tomlkit.load(f))
except Exception:
config_tmpl = self.config
+ config_tmpl["certificates"] = cert_files
+ config_tmpl["project_name"] = project_name
for step, settings in self.config.items():
+ if step in ("certificates", "project_name"):
+ continue
config_tmpl[step]["hosts"] = settings["hosts"]
for key, config in settings["config"].items():
config_tmpl[step]["config"][key] = config
save_file.parent.mkdir(exist_ok=True, parents=True)
- config_tmpl["cert_file"] = cert_file
- config_tmpl["project_name"] = project_name
with open(save_file, "w") as f:
toml_string = tomlkit.dumps(config_tmpl)
f.write(toml_string)
@@ -221,10 +226,9 @@ class MainApp(npyscreen.NPSAppManaged):
"""Read the deployment-steps from the cache."""
return cast(List[str], self._read_cache("steps", ["core", "web", "db", "solr"]))
- @property
- def cert_file(self) -> str:
+ def read_cert_file(self, key: str) -> str:
"""Read the certificate file from the cache."""
- return cast(str, self._read_cache("cert_file", ""))
+ return cast(str, self._read_cache(key, ""))
@property
def save_file(self) -> str:
diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py
index 751917c12f6a5980a697fc8e610a988eb02ac842..6b7c177a1d349fa803925c79e3eb38de291f2ed0 100644
--- a/src/freva_deployment/utils.py
+++ b/src/freva_deployment/utils.py
@@ -5,7 +5,6 @@ import logging
import json
from pathlib import Path
import re
-import shlex
from subprocess import run, PIPE
from typing import cast, NamedTuple
@@ -18,14 +17,14 @@ import toml
logging.basicConfig(format="%(name)s - %(levelname)s - %(message)s", level=logging.INFO)
logger = logging.getLogger("freva-deployment")
-RichConsole = Console(style="bold", markup=True, force_terminal=True)
+RichConsole = Console(markup=True, force_terminal=True)
config_dir = Path(appdirs.user_config_dir()) / "freva" / "deployment"
asset_dir = Path(appdirs.user_data_dir()) / "freva" / "deployment"
password_prompt = (
- "Choose a [b]master password[/], this password will be used to:\n"
+ "[green]Choose[/] a [b]master password[/], this password will be used to:\n"
"- create a self signed [magenta]certificate[/] for accessing the freva credentials\n"
"- create the [magenta]mysql root[/] password\n"
- "- set the [magenta]django admin[/] web password\n[b]enter master password[/]"
+ "- set the [magenta]django admin[/] web password\n[b][green]choose[/] master password[/]"
)
ServiceInfo = NamedTuple(
@@ -181,7 +180,7 @@ def get_passwd(min_characters: int = 8) -> str:
return _create_passwd(min_characters, msg)
except ValueError as e:
RichConsole.print(f"[red]{e.__str__()}[/]")
- msg = "[b]re-enter[/] master password"
+ msg = "[b red]re-enter[/] master password"
def _create_passwd(min_characters: int, msg: str = "") -> str:
@@ -196,81 +195,13 @@ def _create_passwd(min_characters: int, msg: str = "") -> str:
if is_ok is False or is_safe is False:
raise ValueError(
(
- "Password confirm the following constraints:\n"
+ "Password must confirm the following constraints:\n"
f"- {min_characters} alphanumeric characters long,\n"
- "- have both lower and upper case characters,\n"
- "- have at least one special special characters."
+ "- have both, lower and upper case characters,\n"
+ "- have at least one special special character."
)
)
master_pass_2 = Prompt.ask("[bold green]re-enter[/] master password", password=True)
if master_pass != master_pass_2:
raise ValueError("Passwords do not match")
return master_pass
-
-
-def create_self_signed_cert(certfile: Path | str) -> Path:
- """Create a public and private key file.
-
- Parameters:
- ===========
- certfile:
- path of the certificate file.
-
- Returns:
- ========
- pathlib.Path : path to the public certificate file.
-
- """
- public_cert = Path(certfile).with_suffix(".crt")
- private_cert = public_cert.with_suffix(".key")
- public_cert.parent.mkdir(exist_ok=True, parents=True)
- logger.info(f"Creating Self Signed Certificate file: {public_cert}")
- msg = """You are about to be asked to enter information that will be incorporated
-into your certificate request.
-What you are about to enter is what is called a Distinguished Name or a DN.
-There are quite a few fields but you can leave some blank
-For some fields there will be a default value,
-If you enter '.', the field will be left blank."""
- print(msg)
- for file in private_cert, public_cert:
- try:
- file.unlink()
- except FileNotFoundError:
- pass
- defaults = dict(
- C="DE",
- ST="Hamburg",
- L="Hamburg",
- O="Deutsches Klimarechenzentrum GmbH",
- OU="DM",
- CN="freva.dkrz.de",
- emailAddress="freva@dkrz.de",
- )
- steps = dict(
- C=input(f'Country Name (2 letter code) [{defaults["C"]}]: '),
- ST=input(f'State or Province Name (full name) [{defaults["ST"]}]: '),
- L=input(f'Locality Name (eg, city) [{defaults["L"]}]: '),
- O=input(f'Organization Name (eg, company) [{defaults["O"]}]: '),
- OU=input(f'Organization Unit Name (eg, section) [{defaults["OU"]}]: '),
- CN=input(
- f'Common Name (eg, e.g. server FQDN or YOUR name) [{defaults["CN"]}]: '
- ),
- emailAddress=input(
- (
- "Email Address (eg, e.g. server FQDN or YOUR name) "
- f'[{defaults["emailAddress"]}]: '
- )
- ),
- )
- sub_j = "/".join(f"{k}={v.strip() or defaults[k]}" for (k, v) in steps.items())
- cmd = (
- f"openssl req -newkey rsa:4096 -x509 -sha256 -nodes -out {public_cert}"
- f' -keyout {private_cert} -subj "/{sub_j} "'
- )
- run(shlex.split(cmd), stdout=PIPE, stderr=PIPE, check=False)
- for file in private_cert, public_cert:
- if not file.is_file():
- raise FileNotFoundError(
- f"Certificate creation failed, check the command:\n {cmd}"
- )
- return public_cert.absolute()