Changed command line handling, added tests, and fixed a message in script generator

* Added setting of sectioned variables via command line, using '.' as section path separator and '..' for verbatim periods
* Added automated testing using Python's unittest library
* Disambiguated warning due to missing experiment configuration; now mentions 'experiment'. 

Makefile

@@ -9,6 +9,9 @@ distclean: clean
 
 doc:
 
+check:
+	python test.py -v
+
 dist: __FORCE__
 	python setup.py sdist
 

expconfig.py

@@ -127,8 +127,8 @@ class ExpConfig(ConfigObj):
             config_versions.append(pre_config['VERSION_'])
             del pre_config['VERSION_']
         else:
-            feedback.warning("cannot find config for '%s', using default only",
-                             experiment_type)
+            feedback.warning("cannot find experiment config for '%s', "+
+                             "using default only", experiment_type)
 
         # Read host environment settings from library
 

mkexp

@@ -184,6 +184,28 @@ def format_namelist(section):
                 lines.write('/\n')
     return lines.getvalue()
 
+def value_to_list(value):
+    result = value.split(',')
+    if len(result) == 1:
+        result = result[0]
+    return result
+
+def assign_to_dict(assign):
+    (key, value) = assign.split('=', 1)
+    current = value_to_list(value)
+    pos = key.rfind('.')
+    while(pos >= 0):
+        if key[pos-1] == '.':
+            key = key[0:pos-1]+key[pos:]
+            pos = key.rfind('.', 0, pos-1)
+        else:
+            current = {key[pos+1:]: current}
+            key = key[0:pos]
+            pos = key.rfind('.')
+    if key:
+        current = {key: current}
+    return current
+
 #
 # Main routine
 #
@@ -239,7 +261,9 @@ template_env.filters['filter'] = lambda x, f=None: filter(f, x)
 # Read and store configuration info from input and experiments' library
 # Store environment as default for control settings, then add config from files
 
-extra_dict = dict(map(lambda x: x.split('=', 1), args.assigns))
+extra_dict = {}
+for assign in args.assigns:
+    extra_dict.update(assign_to_dict(assign))
 extra_dict.update(mkexp_input='Generated by $$Id$$')
 try:
     config = ExpConfig(experiment_config_name, extra_dict, config_roots)

test.py

+import os
+import re
+import sys
+import unittest
+
+def align(string):
+    return re.sub(r'\n\s*', '\n', string.lstrip())
+
+def script(string):
+    return align("""
+        set -e
+        unset CDPATH
+        cd test
+        PATH=..:$PATH
+    """) + align(string)
+
+class RunningTestCase(unittest.TestCase):
+
+    script_clean = script("""
+        rm -rf experiments
+    """)
+
+    script_run = script("""
+        mkexp test0001.config
+    """)
+
+    def setUp(self):
+        os.system(self.script_clean)
+
+    def test_missing_config_file(self):
+        (ignore, output) = os.popen4('./mkexp')
+        result = output.read()
+        self.assertIn('error: too few arguments', result)
+
+    def test_clean_run(self):
+        expected = align("""
+            Hey: cannot find experiment config for '', using default only
+            Script directory: 'experiments/test0001'
+            Data directory: 'experiments/test0001'
+        """)
+        (ignore, output) = os.popen4(self.script_run)
+        result = output.read()
+        self.assertMultiLineEqual(expected, result)
+
+    def test_backup_run(self):
+        expected = "Hey: script directory already exists, "+\
+                   "moving existing scripts to backup"
+        (ignore, output) = os.popen4(self.script_run)
+        ignore = output.read()
+        (ignore, output) = os.popen4(self.script_run)
+        result = output.read()
+        self.assertIn(expected, result)
+
+    def test_pass_section_variable(self):
+        script_section = script("""
+            mkexp test0001.config \
+                namelists.namelist..echam.runctl.dt_start=2345,01,23,12,34,56
+        """)
+        expected = "dt_start = 2345, 1, 23, 12, 34, 56"
+        (ignore, output) = os.popen4(script_section)
+        ignore = output.read()
+        output = open('test/experiments/test0001/test0001.run')
+        result = output.read()
+        self.assertIn(expected, result)
+
+    @classmethod
+    def tearDownClass(cls):
+        os.system(cls.script_clean)
+
+if __name__ == '__main__':
+    unittest.main()

test/standard_environments/DEFAULT.config

+VERSION_ = $$Id: New Environment $$

test/standard_experiments/DEFAULT.config

+VERSION_ = $$Id: New Experiment $$
+
+SCRIPT_DIR = experiments/$EXP_ID
+DATA_DIR = $SCRIPT_DIR
+WORK_DIR = $SCRIPT_DIR

test/test0001.config

+EXP_TYPE =
+# ENVIRONMENT = # empty ENVIRONMENT does _not_ load default!
+
+VAR1 = value1
+VAR2 = $VAR1
+VAR3 = $$VAR1
+
+[jobs]
+  [[run]]
+    time_limit = 00:30:00
+
+[namelists]
+  [[namelist.echam]]
+    [[[runctl]]]
+      dt_start = 1849, 12, 31, 00, 00, 00

test/test0001.run.tmpl

+#
+# %{EXP_ID}.%{JOB.id} - %{mkexp_input}
+#
+# %{VERSIONS_|join('\n# ')}
+#
+# time_limit = %{JOB.time_limit}
+#
+
+EXP_ID=%{EXP_ID}
+EXP_TYPE=%{EXP_TYPE}
+ENVIRONMENT=%{ENVIRONMENT}
+
+VAR1=%{VAR1}
+VAR2=%{VAR2}
+VAR3=%{VAR3}
+
+cat << ___ > namelist.echam
+%{NAMELIST_ECHAM}
+___