Changed handling of blank environment, fixed section handling on command line,...

Changed handling of blank environment, fixed section handling on command line, cleaned up testing, added release changes

* Changed behavior if environment name is set but empty: now uses default instead of looking for '' environment
* Fixed command line handling: different variables in the same section are now all recognized (see IssueID #5501). The test cases are updated accordingly
* Re-organized test cases to group command line handling in its own class
* Added empty [namelists] section to test config, simplifying test writing
* Added CHANGES.txt for upcoming release 0.3.0, updated setup to 0.3.0dev

CHANGES.txt

+=================
+Make Experiments!
+=================
+
+---------------
+Release Changes
+---------------
+
+Release 0.3.0
+=============
+
+Global
+------
+
+* Support for using native variables in templates
+
+  * Detection of variable use in configuration keys and values
+  * Context query for global template variable values
+  * Native variable format as special job configuration (.var_format)
+
+* Backup of old versions for re-generated files
+
+  * Backup directory with timestamp for each re-generation, containing previous
+    files incl. README
+  * Fixed 'diffexp' to ignore backup directories
+
+* Support for out-of-directory generation
+
+  * Added command line option and path environment variable for configuration
+    files (--path, MPEXP_PATH)
+  * Changed template lookup paths to also use these settings
+
+* Extended queue settings to host environment
+
+  * Renamed configuration variable and directory (QUEUE_TYPE -> ENVIRONMENT,
+    standard_queue_settings -> standard_environments)
+  * Support for setting environment on the command line
+  * Support for default or blank host environment
+
+* Support for setting section variables on the command line
+
+* Fixed 'diffexp' to ignore CDPATH settings
+
+* Added model directory, verbose mode to 'getexp'
+
+Configuration
+-------------
+
+* Support for job inheritance (.extends), and job removal (.remove)
+
+  * Inheritance of job configuration
+  * Namelist overloading
+
+* Support for packages of configuration options (EXP_OPTIONS, standard_options)
+
+* Improved namelist handling
+
+  * Support for removing namelist groups and variables from the parent
+    configuration (.remove)
+  * Support for namelist templates for non-Fortran namelists, e.g. OASIS
+    namcouple (.use_template)
+  * Namelist output restricted to 80 characters per line
+  * Support for automatic conversion of comments in namelist configurations,
+    on all levels
+  * Support for example namelist settings (commented assignments)
+
+* Extended expressions in configuration files
+
+  * Use of lists in configuration expressions
+  * Support for reading configuration values from a file (read)
+  * Second to time string conversion in configuration files (sec2time)
+
+* Support for overriding the 'tasks' job variable
+
+* Fixed evaluation of job configuration levels (global settings were masking
+  user settings)
+
+* Experiment identifier is taken from the file name by default
+
+* Changed to mask dollar characters when reading environment variables into
+  configuration to avoid spurious interpolation
+
+Templates
+---------
+
+* Support for input file template library
+
+  * 'split' filter for use in templates
+  * 'filter' filter for use in templates
+  * 'wrapstring' work-around for legacy Jinja (2.6)
+
+* Support for user defined, experiment specific templates
+
+* Support for 'mkexp' version info in templates (mkexp_input)
+
+

MANIFEST.in

-include LICENSE.txt
\ No newline at end of file
+include LICENSE.txt CHANGES.txt

expconfig.py

@@ -124,9 +124,12 @@ class ExpConfig(ConfigObj):
         pre_config = ConfigObj(experiment_config_name, interpolation=False)
 
         experiment_type = extra_dict.get('EXP_TYPE', pre_config['EXP_TYPE'])
+        # Empty environment should load default
         environment = extra_dict.get('ENVIRONMENT', 
                       pre_config.get('ENVIRONMENT',
                       ExpConfig.default_name))
+        if not environment:
+            environment = ExpConfig.default_name
         # Options should always be treated as a list
         options = extra_dict.get('EXP_OPTIONS',
                   pre_config.get('EXP_OPTIONS',
@@ -211,6 +214,7 @@ class ExpConfig(ConfigObj):
         # Add extra dictionary
         pre_config.merge(extra_dict)
 
+        # Backwards compatibility ENVIRONMENT -> QUEUE_TYPE
         if not 'ENVIRONMENT' in pre_config:
             pre_config['ENVIRONMENT'] = environment
 

mkexp

@@ -309,10 +309,10 @@ template_env.filters['filter'] = lambda x, f=None: filter(f, x)
 # Read and store configuration info from input and experiments' library
 # Store environment as default for control settings, then add config from files
 
-extra_dict = {}
+extra_dict = ConfigObj(interpolation=False)
 for assign in args.assigns:
-    extra_dict.update(assign_to_dict(assign))
-extra_dict.update(mkexp_input='Generated by $$Id$$')
+    extra_dict.merge(assign_to_dict(assign))
+extra_dict['mkexp_input'] = 'Generated by $$Id$$'
 try:
     config = ExpConfig(experiment_config_name, extra_dict, config_roots)
 except ExpConfigError as error:

setup.py

 from distutils.core import setup
 
 name = 'mkexp'
-version = '0.2.1dev'
+version = '0.3.0dev'
 
 setup(
     name = name,

test.py

@@ -80,17 +80,25 @@ class RunningTestCase(MkexpTestCase):
         result = output(self.script_run)
         self.assertIn(expected, result)
 
-class ContentTestCase(MkexpTestCase):
+class CommandLineTestCase(MkexpTestCase):
 
-    def test_pass_section_variable(self):
+    def test_pass_section_variables(self):
         script_section = script("""
             mkexp test0001.config \
-                namelists.namelist..echam.runctl.dt_start=2345,01,23,12,34,56
+                namelists.namelist..echam.runctl.dt_start=2345,01,23,12,34,56 \
+                namelists.namelist..echam.runctl.some_file=abcdefgh.ijk
         """)
-        expected = "dt_start = 2345, 1, 23, 12, 34, 56"
+        expecteds = ["dt_start = 2345, 1, 23, 12, 34, 56",
+                     "some_file = 'abcdefgh.ijk'"]
         ignore = output(script_section)
         result = readfile('test/experiments/test0001/test0001.run')
-        self.assertIn(expected, result)
+        for expected in expecteds:
+            self.assertIn(expected, result)
+
+    def test_pass_new_job(self):
+        output(script("mkexp test0001.config jobs.dummy...extends=run"))
+        readfile('test/experiments/test0001/test0001.dummy')
+        # Should exist, otherwise exception is thrown
 
     def test_options(self):
         script_option = script("""
@@ -101,11 +109,12 @@ class ContentTestCase(MkexpTestCase):
         result = readfile('test/experiments/test0001/test0001.run')
         self.assertIn(expected, result)
 
+class ContentTestCase(MkexpTestCase):
+
     def test_job_override(self):
         exp_id = "test_job_override"
         writeconfig(exp_id, """
             EXP_TYPE =
-            [namelists]
             [jobs]
               key1 = global
               key2 = global
@@ -169,7 +178,6 @@ class ContentTestCase(MkexpTestCase):
             GLOBAL2 = $${VAR2} # Uninitialized
             GLOBAL3 = $${VAR1} # Used twice, may only be listed once
             GLOBAL${FOUR} = 4  # (Uninitialized) Variable in key
-            [namelists]
             [jobs]
               [["""+job_id+"""]]
         """)

test/standard_experiments/DEFAULT.config

@@ -3,3 +3,5 @@ VERSION_ = $$Id: New Experiment $$
 SCRIPT_DIR = experiments/$EXP_ID
 DATA_DIR = $SCRIPT_DIR
 WORK_DIR = $SCRIPT_DIR
+
+[namelists]