From 6ff6ef9c41b9b762861f46b8d02adfec66f5b0eb Mon Sep 17 00:00:00 2001
From: Karl-Hermann Wieners <karl-hermann.wieners@mpimet.mpg.de>
Date: Wed, 11 Feb 2015 12:52:26 +0000
Subject: [PATCH] Changed handling of blank environment, fixed section handling
on command line, cleaned up testing, added release changes
* Changed behavior if environment name is set but empty: now uses default instead of looking for '' environment
* Fixed command line handling: different variables in the same section are now all recognized (see IssueID #5501). The test cases are updated accordingly
* Re-organized test cases to group command line handling in its own class
* Added empty [namelists] section to test config, simplifying test writing
* Added CHANGES.txt for upcoming release 0.3.0, updated setup to 0.3.0dev
---
CHANGES.txt | 96 ++++++++++++++++++++++++
MANIFEST.in | 2 +-
expconfig.py | 4 +
mkexp | 6 +-
setup.py | 2 +-
test.py | 22 ++++--
test/standard_experiments/DEFAULT.config | 2 +
7 files changed, 122 insertions(+), 12 deletions(-)
create mode 100644 CHANGES.txt
diff --git a/CHANGES.txt b/CHANGES.txt
new file mode 100644
index 0000000..9c9ab19
--- /dev/null
+++ b/CHANGES.txt
@@ -0,0 +1,96 @@
+=================
+Make Experiments!
+=================
+
+---------------
+Release Changes
+---------------
+
+Release 0.3.0
+=============
+
+Global
+------
+
+* Support for using native variables in templates
+
+ * Detection of variable use in configuration keys and values
+ * Context query for global template variable values
+ * Native variable format as special job configuration (.var_format)
+
+* Backup of old versions for re-generated files
+
+ * Backup directory with timestamp for each re-generation, containing previous
+ files incl. README
+ * Fixed 'diffexp' to ignore backup directories
+
+* Support for out-of-directory generation
+
+ * Added command line option and path environment variable for configuration
+ files (--path, MPEXP_PATH)
+ * Changed template lookup paths to also use these settings
+
+* Extended queue settings to host environment
+
+ * Renamed configuration variable and directory (QUEUE_TYPE -> ENVIRONMENT,
+ standard_queue_settings -> standard_environments)
+ * Support for setting environment on the command line
+ * Support for default or blank host environment
+
+* Support for setting section variables on the command line
+
+* Fixed 'diffexp' to ignore CDPATH settings
+
+* Added model directory, verbose mode to 'getexp'
+
+Configuration
+-------------
+
+* Support for job inheritance (.extends), and job removal (.remove)
+
+ * Inheritance of job configuration
+ * Namelist overloading
+
+* Support for packages of configuration options (EXP_OPTIONS, standard_options)
+
+* Improved namelist handling
+
+ * Support for removing namelist groups and variables from the parent
+ configuration (.remove)
+ * Support for namelist templates for non-Fortran namelists, e.g. OASIS
+ namcouple (.use_template)
+ * Namelist output restricted to 80 characters per line
+ * Support for automatic conversion of comments in namelist configurations,
+ on all levels
+ * Support for example namelist settings (commented assignments)
+
+* Extended expressions in configuration files
+
+ * Use of lists in configuration expressions
+ * Support for reading configuration values from a file (read)
+ * Second to time string conversion in configuration files (sec2time)
+
+* Support for overriding the 'tasks' job variable
+
+* Fixed evaluation of job configuration levels (global settings were masking
+ user settings)
+
+* Experiment identifier is taken from the file name by default
+
+* Changed to mask dollar characters when reading environment variables into
+ configuration to avoid spurious interpolation
+
+Templates
+---------
+
+* Support for input file template library
+
+ * 'split' filter for use in templates
+ * 'filter' filter for use in templates
+ * 'wrapstring' work-around for legacy Jinja (2.6)
+
+* Support for user defined, experiment specific templates
+
+* Support for 'mkexp' version info in templates (mkexp_input)
+
+
diff --git a/MANIFEST.in b/MANIFEST.in
index 1e0fc79..c29cdd7 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1 +1 @@
-include LICENSE.txt
\ No newline at end of file
+include LICENSE.txt CHANGES.txt
diff --git a/expconfig.py b/expconfig.py
index c424482..faf8ea7 100644
--- a/expconfig.py
+++ b/expconfig.py
@@ -124,9 +124,12 @@ class ExpConfig(ConfigObj):
pre_config = ConfigObj(experiment_config_name, interpolation=False)
experiment_type = extra_dict.get('EXP_TYPE', pre_config['EXP_TYPE'])
+ # Empty environment should load default
environment = extra_dict.get('ENVIRONMENT',
pre_config.get('ENVIRONMENT',
ExpConfig.default_name))
+ if not environment:
+ environment = ExpConfig.default_name
# Options should always be treated as a list
options = extra_dict.get('EXP_OPTIONS',
pre_config.get('EXP_OPTIONS',
@@ -211,6 +214,7 @@ class ExpConfig(ConfigObj):
# Add extra dictionary
pre_config.merge(extra_dict)
+ # Backwards compatibility ENVIRONMENT -> QUEUE_TYPE
if not 'ENVIRONMENT' in pre_config:
pre_config['ENVIRONMENT'] = environment
diff --git a/mkexp b/mkexp
index f05492e..a31b7bc 100755
--- a/mkexp
+++ b/mkexp
@@ -309,10 +309,10 @@ template_env.filters['filter'] = lambda x, f=None: filter(f, x)
# Read and store configuration info from input and experiments' library
# Store environment as default for control settings, then add config from files
-extra_dict = {}
+extra_dict = ConfigObj(interpolation=False)
for assign in args.assigns:
- extra_dict.update(assign_to_dict(assign))
-extra_dict.update(mkexp_input='Generated by $$Id$$')
+ extra_dict.merge(assign_to_dict(assign))
+extra_dict['mkexp_input'] = 'Generated by $$Id$$'
try:
config = ExpConfig(experiment_config_name, extra_dict, config_roots)
except ExpConfigError as error:
diff --git a/setup.py b/setup.py
index 1633aaa..a9b0b8c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
from distutils.core import setup
name = 'mkexp'
-version = '0.2.1dev'
+version = '0.3.0dev'
setup(
name = name,
diff --git a/test.py b/test.py
index a78dc16..86b47e8 100644
--- a/test.py
+++ b/test.py
@@ -80,17 +80,25 @@ class RunningTestCase(MkexpTestCase):
result = output(self.script_run)
self.assertIn(expected, result)
-class ContentTestCase(MkexpTestCase):
+class CommandLineTestCase(MkexpTestCase):
- def test_pass_section_variable(self):
+ def test_pass_section_variables(self):
script_section = script("""
mkexp test0001.config \
- namelists.namelist..echam.runctl.dt_start=2345,01,23,12,34,56
+ namelists.namelist..echam.runctl.dt_start=2345,01,23,12,34,56 \
+ namelists.namelist..echam.runctl.some_file=abcdefgh.ijk
""")
- expected = "dt_start = 2345, 1, 23, 12, 34, 56"
+ expecteds = ["dt_start = 2345, 1, 23, 12, 34, 56",
+ "some_file = 'abcdefgh.ijk'"]
ignore = output(script_section)
result = readfile('test/experiments/test0001/test0001.run')
- self.assertIn(expected, result)
+ for expected in expecteds:
+ self.assertIn(expected, result)
+
+ def test_pass_new_job(self):
+ output(script("mkexp test0001.config jobs.dummy...extends=run"))
+ readfile('test/experiments/test0001/test0001.dummy')
+ # Should exist, otherwise exception is thrown
def test_options(self):
script_option = script("""
@@ -101,11 +109,12 @@ class ContentTestCase(MkexpTestCase):
result = readfile('test/experiments/test0001/test0001.run')
self.assertIn(expected, result)
+class ContentTestCase(MkexpTestCase):
+
def test_job_override(self):
exp_id = "test_job_override"
writeconfig(exp_id, """
EXP_TYPE =
- [namelists]
[jobs]
key1 = global
key2 = global
@@ -169,7 +178,6 @@ class ContentTestCase(MkexpTestCase):
GLOBAL2 = $${VAR2} # Uninitialized
GLOBAL3 = $${VAR1} # Used twice, may only be listed once
GLOBAL${FOUR} = 4 # (Uninitialized) Variable in key
- [namelists]
[jobs]
[["""+job_id+"""]]
""")
diff --git a/test/standard_experiments/DEFAULT.config b/test/standard_experiments/DEFAULT.config
index 3e4682b..e201216 100644
--- a/test/standard_experiments/DEFAULT.config
+++ b/test/standard_experiments/DEFAULT.config
@@ -3,3 +3,5 @@ VERSION_ = $$Id: New Experiment $$
SCRIPT_DIR = experiments/$EXP_ID
DATA_DIR = $SCRIPT_DIR
WORK_DIR = $SCRIPT_DIR
+
+[namelists]
--
GitLab