Changed script generation for easier job setup.

* Changed job extension to also inherit job settings (used to inherit script only)
* Changed job extension to allow namelist overwriting per job
* Namelist overwriting requires copying of the configuration data; this should be revisited

expconfig.py

@@ -148,14 +148,14 @@ class ExpConfig(ConfigObj):
         pre_config = None
 
         config_lines.seek(0)
-        ConfigObj.__init__(self, config_lines, interpolation='template')
+        pre_config = ConfigObj(config_lines, interpolation='template')
 
         # Extract experiment description from initial comment
         # if not set explicitly
-        if not self.has_key('EXP_DESCRIPTION'):
+        if not pre_config.has_key('EXP_DESCRIPTION'):
             is_empty = lambda s: re.match(r'^\s*$', s)
             rm_comment = lambda s: re.sub(r'^\s*# ?', '', s)       
-            self['EXP_DESCRIPTION'] = "\n".join(
+            pre_config['EXP_DESCRIPTION'] = "\n".join(
                 reversed(list(
                     dropwhile(is_empty,
                         reversed(list(
@@ -231,7 +231,20 @@ class ExpConfig(ConfigObj):
                 raise ExpConfigError(error.message, key)
             section[key] = value
 
-        self.walk(eval_key)
+        pre_config.walk(eval_key)
+
+        # Re-read final config without interpolation.
+        # This allows copying data without evaluation of version keywords.
+
+        config_lines.seek(0)
+        config_lines.truncate()
+
+        pre_config.write(config_lines)
+        pre_config = None
+
+        config_lines.seek(0)
+        ConfigObj.__init__(self, config_lines, interpolation=False)
+        
         self.experiment_id = self['EXP_ID']
         self.experiment_kind = re.sub(r'-\w+$', '', experiment_type)
 

mkexp

@@ -11,6 +11,7 @@ import stat
 import StringIO
 import sys
 
+from configobj import ConfigObj
 from jinja2 import Environment, FileSystemLoader
 
 from expconfig import ExpConfig, ExpConfigError
@@ -195,11 +196,6 @@ if work_dir != data_dir:
     else:
         feedback.warning("work directory already exists")
 
-# Prepare namelists for inclusion in scripts
-for namelist, groups in config['namelists'].iteritems():
-    namelist_name = re.sub(r'\W', '_', namelist.upper())
-    config[namelist_name] = format_namelist(groups)
-
 # Run template expansion
 
 # Process scripts for each job
@@ -218,18 +214,45 @@ for key, value in config['jobs'].iteritems():
             job_dict[key] = value
         del config['jobs'][key]
 
+# Save configuration to buffer
+config_lines = StringIO.StringIO()
+config.write(config_lines)
+
 # Paste them into each job
 for subjob, subconfig in config['jobs'].iteritems():
     if not subjob in remove_list:
-        subconfig.merge(job_dict)
-        subconfig['id'] = subjob
+
+        # Allow simple inheritance
         template_job = subjob
         if 'extends' in subconfig:
             template_job = subconfig['extends']
+            backup = subconfig.dict()
+            subconfig.merge(config['jobs'][template_job])
+            subconfig.merge(backup) 
+
+        # Add global job settings
+        subconfig.merge(job_dict)
+
+        # Add useful settings
+        subconfig['id'] = subjob
         subconfig['tasks'] = int(subconfig.get('nodes', 1)) * \
                              int(subconfig.get('tasks_per_node', 1))
-        config['JOB'] = subconfig
-        expand_template(config,
+
+        # Check namelist override
+        config_lines.seek(0)
+        job_config = ConfigObj(config_lines, interpolation=False)
+        if 'namelists' in subconfig:
+            job_config['namelists'].merge(subconfig['namelists'])
+            del subconfig['namelists']
+        job_config['JOB'] = subconfig
+
+        # Prepare namelists for inclusion in scripts
+        for namelist, groups in job_config['namelists'].iteritems():
+            namelist_name = re.sub(r'\W', '_', namelist.upper())
+            job_config[namelist_name] = format_namelist(groups)
+
+        # Generate job script
+        expand_template(job_config,
                         get_template_name(config.experiment_kind, template_job),
                         get_script_name(config.experiment_id, subjob))