Changed new script generation

* Added handling of experiment description
** Initial comment is parsed if no EXP_DESCRIPTION is set
** Description is written to script directory as README file
* Added setting of top-level settings from command line

Makefile

 name = mkexp
-version = 0.1.0
+version = 0.1.1dev
 package = $(name)-$(version)
 
 prefix = /usr/local

diffexp

@@ -13,16 +13,17 @@ die () {
 }
 
 [ "x$2" = x ] && die "Oops: invalid number of parameters
-Usage: $PROGRAM config_a config_b"
+Usage: $PROGRAM config_a config_b [key=value...]"
 
 CONFIG_A=$1
 CONFIG_B=$2
+shift; shift
 
-eval `getexp "$CONFIG_A" || echo \; exit $?`
+eval `getexp "$CONFIG_A" "$@" || echo \; exit $?`
 EXP_A=$EXP_ID
 PATH_A=$SCRIPT_DIR
 
-eval `getexp "$CONFIG_B" || echo \; exit $?`
+eval `getexp "$CONFIG_B" "$@" || echo \; exit $?`
 EXP_B=$EXP_ID
 PATH_B=$SCRIPT_DIR
 

examples/mean_values.config

 # 
-# ECHAM experiment configuration file
+# Performs a standard $EXP_TYPE simulation, but using monthy mean output.
+#
+# This experiment switches off the usual output and uses the mvstream module
+# to create monthly mean data instead. The selection of variables reflects the
+# standard selection that goes into the ATM, BOT and LOG files.
 #
 
 EXP_ID = mean_values

examples/old_input_data.config

 # 
-# ECHAM experiment configuration file
+# Performs a standard $EXP_TYPE simulation, using the original input data.
+#
+# Again, this is quite a simple setup, like reference, but for this experiment
+# we use another input directory containing the original CMIP5 input data.
 #
 
 EXP_ID = old_input_data

examples/reference.config

 # 
-# ECHAM experiment configuration file
+# Performs a standard $EXP_TYPE simulation.
+#
+# This is an example for a very simple setup, changing only the model location,
+# queue settings and accounting information.
 #
 
 EXP_ID = reference

expconfig.py

@@ -8,6 +8,8 @@ import os
 import re
 import StringIO
 
+from itertools import dropwhile
+
 from configobj import ConfigObj, InterpolationError
 
 import feedback
@@ -35,7 +37,7 @@ class ExpConfig(ConfigObj):
 
     # Class constructor
 
-    def __init__(self, experiment_config_name):
+    def __init__(self, experiment_config_name, extra_dict={}):
         '''Read experiment config to get basic settings
         
         TODO: probably nicer if default experiment is given as argument
@@ -56,9 +58,7 @@ class ExpConfig(ConfigObj):
 
         pre_config = ConfigObj(experiment_config_name, interpolation=False)
 
-        self.experiment_id = pre_config['EXP_ID']
-        experiment_type = pre_config['EXP_TYPE']
-        self.experiment_kind = re.sub(r'-\wR$', '', experiment_type)
+        experiment_type = extra_dict.get('EXP_TYPE', pre_config['EXP_TYPE'])
 
         pre_config = None
 
@@ -91,12 +91,16 @@ class ExpConfig(ConfigObj):
 
         # Re-read config to allow overriding default settings
         # TODO: probably nicer if default experiment is given as argument
-        pre_config.merge(ConfigObj(experiment_config_name, interpolation=False))
+        experiment_config = ConfigObj(experiment_config_name, interpolation=False)
+        pre_config.merge(experiment_config)
         split_jobs(pre_config)
 
         # Add complete versioning info
         pre_config['VERSIONS_'] = ', '.join(config_versions)
 
+        # Add extra dictionary
+        pre_config.merge(extra_dict)
+
         # Re-read merged config with interpolation set
         # This works around incomprehensible inheritance of interpolation with merge
         # Then make sure that all values are interpolated
@@ -109,6 +113,22 @@ class ExpConfig(ConfigObj):
         config_lines.seek(0)
         ConfigObj.__init__(self, config_lines, interpolation='template')
 
+        if not self.has_key('EXP_DESCRIPTION'):
+            is_empty = lambda s: re.match(r'^\s*$', s)
+            rm_comment = lambda s: re.sub(r'^\s*# ?', '', s)       
+            self['EXP_DESCRIPTION'] = "\n".join(
+                reversed(list(
+                    dropwhile(is_empty,
+                        reversed(list(
+                            dropwhile(is_empty,
+                                map(rm_comment,
+                                    experiment_config.initial_comment)
+                            )
+                        )) 
+                    )
+                ))
+            )
+            
         def eval_key(section, key):
             try:
                 section[key] = section[key]
@@ -116,4 +136,6 @@ class ExpConfig(ConfigObj):
                 raise ExpConfigError(error.message, key)
 
         self.walk(eval_key)
+        self.experiment_id = self['EXP_ID']
+        self.experiment_kind = re.sub(r'-\wR$', '', experiment_type)
 

feedback.py

@@ -14,6 +14,6 @@ warning = logging.warning
 error = logging.error
 critical = logging.critical
 
-def die(message):
+def die(message, status=1):
     error(message)
-    sys.exit(1)
+    sys.exit(status)

getexp

@@ -17,21 +17,31 @@ from feedback import die
 
 # Check command line
 
-if len(sys.argv) != 2:
-    die('invalid number of parameters\n' +
-        'Usage: '+sys.argv[0]+' experiment_config_name')
+program = sys.argv.pop(0)
+usage = 'Usage: ' + program + ' experiment_config_name [key=value...]'
+
+if len(sys.argv) < 1:
+    feedback.die('invalid number of parameters\n' + usage)
+
+experiment_config_name = sys.argv.pop(0)
 
-experiment_config_name = sys.argv[1]
 if not os.path.exists(experiment_config_name):
-    die("config file '{0}' does not exist".format(experiment_config_name))
+    feedback.die("config file '{0}' does not exist".format(experiment_config_name))
+
+invalid_args = filter(lambda x: not x.find('=')+1, sys.argv)
+
+if invalid_args:
+    feedback.die("invalid parameters ('"+"', '".join(invalid_args)+"')\n" +
+                 usage)
 
 # Read and store configuration info from input and experiments' library
 # Store environment as default for control settings, then add config from files
 
 try:
-    config = ExpConfig(experiment_config_name)
+    config = ExpConfig(experiment_config_name,
+                       dict(map(lambda x: x.split('=', 1), sys.argv)))
 except ExpConfigError as error:
-    die(error.message)
+    die(error.message, 2)
 
 print("EXP_ID='{0}'".format(config.experiment_id))
 print("SCRIPT_DIR='{0}'".format(config['SCRIPT_DIR']))

mkexp

@@ -119,24 +119,31 @@ def format_namelist(section):
 
 # Check command line
 
-if len(sys.argv) < 2:
-    feedback.error('invalid number of parameters\n' +
-                  'Usage: '+sys.argv[0]+' experiment_config_name')
-    sys.exit(1)
+program = sys.argv.pop(0)
+usage = 'Usage: ' + program + ' experiment_config_name [key=value...]'
+
+if len(sys.argv) < 1:
+    feedback.die('invalid number of parameters\n' + usage)
+
+experiment_config_name = sys.argv.pop(0)
 
-experiment_config_name = sys.argv[1]
 if not os.path.exists(experiment_config_name):
-    feedback.error("config file '{0}' does not exist".format(experiment_config_name))
-    sys.exit(1)
+    feedback.die("config file '{0}' does not exist".format(experiment_config_name))
+
+invalid_args = filter(lambda x: not x.find('=')+1, sys.argv)
+
+if invalid_args:
+    feedback.die("invalid parameters ('"+"', '".join(invalid_args)+"')\n" +
+                 usage)
 
 # Read and store configuration info from input and experiments' library
 # Store environment as default for control settings, then add config from files
 
 try:
-    config = ExpConfig(experiment_config_name)
+    config = ExpConfig(experiment_config_name,
+                       dict(map(lambda x: x.split('=', 1), sys.argv)))
 except ExpConfigError as error:
-    feedback.error(error.message)
-    sys.exit(2)
+    feedback.die(error.message, 2)
 
 # Create target directories
 
@@ -183,3 +190,9 @@ for subjob, subconfig in config['jobs'].iteritems():
                     get_template_name(config.experiment_kind, template_job),
                     get_script_name(config.experiment_id, subjob))
 
+# Create README file from experiment description
+
+readme_file = open(os.path.join(script_dir, 'README'), 'w')
+readme_file.write(config['EXP_DESCRIPTION'] + '\n')
+readme_file.close()
+

rmexp

@@ -13,13 +13,11 @@ die () {
 }
 
 [ "x$1" = x ] && die "Oops: invalid number of parameters
-Usage: $PROGRAM config_file..."
+Usage: $PROGRAM config_file [key=value...]"
 
-for CONFIG
-do
+CONFIG=$1
+shift
 
-    eval `getexp "$CONFIG" || echo \; exit $?`
+eval `getexp "$CONFIG" "$@" || echo \; exit $?`
 
-    rm -ri $SCRIPT_DIR $WORK_DIR
-
-done
+rm -ri $SCRIPT_DIR $WORK_DIR