From 5e983afdcb145da25e7da334cbb3dbf7ec2d940f Mon Sep 17 00:00:00 2001
From: Karl-Hermann Wieners <karl-hermann.wieners@mpimet.mpg.de>
Date: Mon, 28 Nov 2022 14:26:06 +0100
Subject: [PATCH] Config: add shared namelist settings ([[namelist_a,
namelist_b]])
---
CHANGES.txt | 5 +++
expconfig.py | 46 +++++++++++++++++----------
test.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 122 insertions(+), 17 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 698ff2c..b40c3c4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,11 @@ Release Changes
Release 1.2.1
=============
+Config
+------
+
+* Added shared namelist settings ([[namelist_a, namelist_b]])
+
Release 1.2.0
=============
diff --git a/expconfig.py b/expconfig.py
index d3a833a..6c95dbb 100644
--- a/expconfig.py
+++ b/expconfig.py
@@ -137,18 +137,32 @@ class ExpConfig(ConfigObj):
# Helper functions
#
- def split_jobs(config):
- '''Post-process job definition to allow for shared configs as [[job1, job2]]'''
- if 'jobs' in config:
- sep = re.compile(r'\s*,\s*')
- for subjobs, subconfig in config['jobs'].items():
- if re.search(sep, subjobs):
- for subjob in re.split(sep, subjobs):
- if subjob in config['jobs']:
- config['jobs'][subjob].merge(subconfig.dict())
+ def split_shared_sections(config):
+ '''Process sections to expand shared entries as [[job1, job2]]
+
+ Supports jobs, namelists, and job-specific namelists'''
+
+ sep = re.compile(r'\s*,\s*')
+
+ def split_key(current):
+ for subkey_orig in current.sections:
+ subkeys = re.split(sep, subkey_orig)
+ if len(subkeys) > 1:
+ subconfig = current[subkey_orig]
+ for subkey in subkeys:
+ if subkey in current:
+ current[subkey].merge(subconfig.dict())
else:
- config['jobs'][subjob] = subconfig.dict()
- del config['jobs'][subjobs]
+ current[subkey] = subconfig.dict()
+ del current[subkey_orig]
+
+ if 'namelists' in config.sections:
+ split_key(config['namelists'])
+ if 'jobs' in config.sections:
+ split_key(config['jobs'])
+ for job in config['jobs'].sections:
+ if 'namelists' in config['jobs'][job].sections:
+ split_key(config['jobs'][job]['namelists'])
def get_config_name(lib_name, base_name):
'''Cycle through config path until a match is found.
@@ -479,12 +493,12 @@ class ExpConfig(ConfigObj):
lib_config_name = get_config_name(ExpConfig.exp_lib_dir,
ExpConfig.default_name+'.config')
pre_config.merge(ConfigObj(lib_config_name, interpolation=False))
- split_jobs(pre_config)
+ split_shared_sections(pre_config)
register_version(pre_config, config_versions)
if os.path.exists(setup_config_name):
pre_config.merge(ConfigObj(setup_config_name, interpolation=False))
- split_jobs(pre_config)
+ split_shared_sections(pre_config)
list_assign(pre_config)
register_version(pre_config, config_versions)
@@ -492,7 +506,7 @@ class ExpConfig(ConfigObj):
experiment_type+'.config')
if os.path.exists(lib_config_name):
pre_config.merge(ConfigObj(lib_config_name, interpolation=False))
- split_jobs(pre_config)
+ split_shared_sections(pre_config)
list_assign(pre_config)
register_version(pre_config, config_versions)
else:
@@ -504,7 +518,7 @@ class ExpConfig(ConfigObj):
option+'.config')
if os.path.exists(lib_config_name):
pre_config.merge(ConfigObj(lib_config_name, interpolation=False))
- split_jobs(pre_config)
+ split_shared_sections(pre_config)
list_assign(pre_config)
register_version(pre_config, config_versions)
else:
@@ -530,7 +544,7 @@ class ExpConfig(ConfigObj):
experiment_config = ConfigObj(experiment_config_name,
interpolation=False)
pre_config.merge(experiment_config)
- split_jobs(pre_config)
+ split_shared_sections(pre_config)
list_assign(pre_config)
# Add extra dictionary
diff --git a/test.py b/test.py
index 88a469d..f490fc1 100644
--- a/test.py
+++ b/test.py
@@ -82,12 +82,13 @@ class MkexpSimpleTestCase(MkexpTestCase):
self.job_id = 'job'
MkexpTestCase.setUp(self)
- def run_test(self, template, expected, additional=''):
+ def run_test(self, template, expected, additional='', epilog=''):
writeconfig(self.exp_id, u"""
EXP_TYPE =
"""+additional+u"""
[jobs]
[["""+self.job_id+u"""]]
+ """+epilog+u"""
""")
writetemplate(self.exp_id, self.job_id, template)
expected = align(expected)
@@ -810,6 +811,91 @@ class NamelistInheritanceTestCase(MkexpSimpleTestCase):
[[[group clone]]]
""")
+class KeySplittingTestCase(MkexpSimpleTestCase):
+
+ def test_job_splitting(self):
+ self.run_test(u"""
+ %{{jobs.{job_id}.seniority}}
+ %{{jobs.{job_id}.common}}
+ %{{jobs.job2.seniority}}
+ %{{jobs.job2.common}}
+ """.format(**self.__dict__), u"""
+ elder
+ parents
+ younger
+ parents
+ """, epilog="""
+ seniority = elder
+ [[{job_id}, job2]]
+ common = parents
+ [[job2]]
+ .extends = {job_id}
+ seniority = younger
+ """.format(**self.__dict__))
+
+ def test_namelist_splitting(self):
+ self.run_test(u"""
+ %{NAMELIST_ATM}
+ %{NAMELIST_OCE}
+ """, u"""
+ &group1
+ value = 21
+ /
+ &group2
+ value = 42
+ /
+ &group3
+ value = 84
+ /
+ &group2
+ value = 42
+ /
+ """, u"""
+ [namelists]
+ [[NAMELIST_atm]]
+ [[[group1]]]
+ value = 21
+ [[NAMELIST_atm, NAMELIST_oce]]
+ [[[group2]]]
+ value = 42
+ [[NAMELIST_oce]]
+ [[[group3]]]
+ value = 84
+ """)
+
+ def test_job_namelist_splitting(self):
+ self.run_test(u"""
+ %{NAMELIST_ATM}
+ %{NAMELIST_OCE}
+ """, u"""
+ &group1
+ value = 21
+ /
+ &group2
+ value = 42
+ /
+ &group3
+ value = 84
+ /
+ &group2
+ value = 42
+ /
+ """, u"""
+ [namelists]
+ [[NAMELIST_atm]]
+ [[[group1]]]
+ value = 21
+ [[NAMELIST_oce]]
+ [[[group3]]]
+ value = 84
+ """, u"""
+ [[[namelists]]]
+ [[[[NAMELIST_atm, NAMELIST_oce]]]]
+ [[[[[group2]]]]]
+ value = 42
+ """)
+
+
class JinjaTemplateTestCase(MkexpSimpleTestCase):
def test_ignore_blocks(self):
--
GitLab