Added usability features and documentation

* Added README mode to 'getexp'
* Changed 'getexp' output from warning to info
* Added option to skip directory creation for config tests (--no-make-dirs/-m)
* Support for default configuration settings by model build (SETUP.config)
* Added User Guide

CHANGES.txt

@@ -30,7 +30,7 @@ Global
     files (--path, MPEXP_PATH)
   * Changed template lookup paths to also use these settings
 
-* Extended queue settings to host environment
+* Extended queue settings into host environment
 
   * Renamed configuration variable and directory (QUEUE_TYPE -> ENVIRONMENT,
     standard_queue_settings -> standard_environments)
@@ -41,7 +41,13 @@ Global
 
 * Fixed 'diffexp' to ignore CDPATH settings
 
-* Added model directory, verbose mode to 'getexp'
+* Added model directory, verbose mode, README mode to 'getexp'
+
+* Changed 'getexp' output from warning to info
+
+* Added option to skip directory creation for config tests (--no-make-dirs/-m)
+
+* Added User Guide
 
 Configuration
 -------------
@@ -53,6 +59,8 @@ Configuration
 
 * Support for packages of configuration options (EXP_OPTIONS, standard_options)
 
+* Support for default configuration settings by model build (SETUP.config)
+
 * Improved namelist handling
 
   * Support for removing namelist groups and variables from the parent

Makefile

@@ -6,16 +6,20 @@ clean:
 	$(RM) -r build dist MANIFEST *.pyc
 
 distclean: clean
+	$(RM) doc/*.pdf
 
-doc:
+doc: doc/mkexp.pdf
 
 check:
 	python test.py -v
 
-dist: __FORCE__
+dist:
 	python setup.py sdist
 
 install: all
 	python setup.py install --prefix=$(prefix)
 
-__FORCE__:
+.PHONY: all clean distclean doc check dist install
+
+%.pdf: %.fodt
+	soffice --headless --convert-to pdf --outdir $(@D) $<

expconfig.py

@@ -77,6 +77,7 @@ class ExpConfig(ConfigObj):
     opt_lib_dir = 'standard_options'
     default_name = 'DEFAULT'
     id_name = 'EXP_ID'
+    setup_config_name = 'SETUP.config'
 
     # Class constructor
 
@@ -115,13 +116,110 @@ class ExpConfig(ConfigObj):
                     break
             return config_name
 
+        def read_value(value):
+            if os.path.exists(value):
+                stream = open(value)
+                result = stream.read().strip()
+                stream.close()
+            else:
+                result = ''
+            return result
+
+        def sec2time(seconds):
+            '''Create time string (HH:MM:SS) from second of day'''
+            seconds = int(seconds)
+            if seconds >= 86400:
+                raise ValueError("invalid second of day '{0}'".format(seconds))
+            minutes, s = divmod(seconds, 60)
+            h, m = divmod(minutes, 60)
+            return "{0:02}:{1:02}:{2:02}".format(h, m, s)
+
+        def split_date(value):
+            '''Re-format datetime string to list for use in namelists'''
+            match = re.match(r'^0*(\d+)-0*(\d+)-0*(\d+)'
+                             r'([T ]0*(\d+)(:0*(\d+)(:0*(\d+))?)?)?$', value)
+            if match:
+                numbers = match.groups('0')
+            else:
+                raise ValueError("invalid date/time '{0}'".format(value))
+            return [numbers[i] for i in [0,1,2,4,6,8]]
+
+        def eval_value(value):
+            '''
+                Evaluate key as python expression,
+                return as string or sequence of strings.
+            '''
+            result = eval(value)
+            if isinstance(result, (list, tuple)):
+                result = map(str, result)
+            else:
+                result = str(result)
+            return result
+
+        def eval_value_string(value):
+            '''
+                Evaluate key as python expression,
+                return as string or sequence of strings.
+            '''
+            result = eval_value(value)
+            if isinstance(result, (list, tuple)):
+                result = ", ".join(result)
+            return result
+
+        def eval_expression(value):
+            '''
+                Check if value is a supported expression.
+                If so, evaluate and return result, otherwise just pass through.
+            '''
+            match = re.match(r'^eval\((.*)\)$', value, re.S)
+            if match:
+                return eval_value(match.group(1))
+
+            match = re.match(r'^evals\((.*)\)$', value, re.S)
+            if match:
+                return eval_value_string(match.group(1))
+
+            match = re.match(r'^split_date\((.*)\)$', value, re.S)
+            if match:
+                return split_date(match.group(1))
+
+            match = re.match(r'^sec2time\((.*)\)$', value, re.S)
+            if match:
+                return sec2time(match.group(1))
+
+            match = re.match(r'^read\((.*)\)$', value, re.S)
+            if match:
+                return read_value(match.group(1))
+
+            return value
+
+        # Interpolate and evaluate keys if they are an expression
+        def eval_key(section, key):
+            try:
+                value = section[key]
+                if isinstance(value, (list, tuple)):
+                    value = map(eval_expression, value)
+                elif isinstance(value, basestring):
+                    value = eval_expression(value)                    
+            except (InterpolationError, ValueError) as error:
+                raise ExpConfigError(error.message, key)
+            section[key] = value
+
         #
         # Method body
         #
 
         # Pre-read basic experiment settings
 
-        pre_config = ConfigObj(experiment_config_name, interpolation=False)
+        pre_config = None
+        setup_config_name = get_config_name('', ExpConfig.setup_config_name)
+        if os.path.exists(setup_config_name):
+            pre_config = ConfigObj(setup_config_name, interpolation=False)
+        user_config = ConfigObj(experiment_config_name, interpolation=False)
+        if pre_config:
+            pre_config.merge(user_config)
+        else:
+            pre_config = user_config
 
         experiment_type = extra_dict.get('EXP_TYPE', pre_config['EXP_TYPE'])
         # Empty environment should load default
@@ -143,6 +241,7 @@ class ExpConfig(ConfigObj):
             environment = pre_config['QUEUE_TYPE']
 
         pre_config = None
+        user_config = None
 
         # Start from empty configuration
 
@@ -171,6 +270,12 @@ class ExpConfig(ConfigObj):
         config_versions.append(pre_config['VERSION_'])
         del pre_config['VERSION_']
 
+        if os.path.exists(setup_config_name):
+            pre_config.merge(ConfigObj(setup_config_name, interpolation=False))
+            split_jobs(pre_config)
+            config_versions.append(pre_config['VERSION_'])
+            del pre_config['VERSION_']
+
         lib_config_name = get_config_name(ExpConfig.exp_lib_dir, 
                                           experiment_type+'.config')
         if os.path.exists(lib_config_name):
@@ -251,95 +356,6 @@ class ExpConfig(ConfigObj):
                 ))
             )
 
-        def read_value(value):
-            if os.path.exists(value):
-                stream = open(value)
-                result = stream.read().strip()
-                stream.close()
-            else:
-                result = ''
-            return result
-
-        def sec2time(seconds):
-            '''Create time string (HH:MM:SS) from second of day'''
-            seconds = int(seconds)
-            if seconds >= 86400:
-                raise ValueError("invalid second of day '{0}'".format(seconds))
-            minutes, s = divmod(seconds, 60)
-            h, m = divmod(minutes, 60)
-            return "{0:02}:{1:02}:{2:02}".format(h, m, s)
-
-        def split_date(value):
-            '''Re-format datetime string to list for use in namelists'''
-            match = re.match(r'^0*(\d+)-0*(\d+)-0*(\d+)'
-                             r'([T ]0*(\d+)(:0*(\d+)(:0*(\d+))?)?)?$', value)
-            if match:
-                numbers = match.groups('0')
-            else:
-                raise ValueError("invalid date/time '{0}'".format(value))
-            return [numbers[i] for i in [0,1,2,4,6,8]]
-
-        def eval_value(value):
-            '''
-                Evaluate key as python expression,
-                return as string or sequence of strings.
-            '''
-            result = eval(value)
-            if isinstance(result, (list, tuple)):
-                result = map(str, result)
-            else:
-                result = str(result)
-            return result
-
-        def eval_value_string(value):
-            '''
-                Evaluate key as python expression,
-                return as string or sequence of strings.
-            '''
-            result = eval_value(value)
-            if isinstance(result, (list, tuple)):
-                result = ", ".join(result)
-            return result
-
-        def eval_expression(value):
-            '''
-                Check if value is a supported expression.
-                If so, evaluate and return result, otherwise just pass through.
-            '''
-            match = re.match(r'^eval\((.*)\)$', value, re.S)
-            if match:
-                return eval_value(match.group(1))
-
-            match = re.match(r'^evals\((.*)\)$', value, re.S)
-            if match:
-                return eval_value_string(match.group(1))
-
-            match = re.match(r'^split_date\((.*)\)$', value, re.S)
-            if match:
-                return split_date(match.group(1))
-
-            match = re.match(r'^sec2time\((.*)\)$', value, re.S)
-            if match:
-                return sec2time(match.group(1))
-
-            match = re.match(r'^read\((.*)\)$', value, re.S)
-            if match:
-                return read_value(match.group(1))
-
-            return value
-
-        # Interpolate and evaluate keys if they are an expression
-        def eval_key(section, key):
-            try:
-                value = section[key]
-                if isinstance(value, (list, tuple)):
-                    value = map(eval_expression, value)
-                elif isinstance(value, basestring):
-                    value = eval_expression(value)                    
-            except (InterpolationError, ValueError) as error:
-                raise ExpConfigError(error.message, key)
-            section[key] = value
-
         pre_config.walk(eval_key)
 
         # Re-read final config without interpolation.

feedback.py

@@ -15,6 +15,13 @@ logging.addLevelName(logging.WARNING, 'Hey')
 logging.addLevelName(logging.ERROR, 'Oops')
 logging.addLevelName(logging.CRITICAL, 'Sorry')
 
+DEBUG = logging.DEBUG
+INFO = logging.INFO
+WARNING = logging.WARNING
+ERROR = logging.ERROR
+CRITICAL = logging.CRITICAL
+
+debug = logging.debug
 info = logging.info
 warning = logging.warning
 error = logging.error
@@ -28,3 +35,5 @@ def die(message, *args, **kwargs):
     error(message, *args)
     sys.exit(status)
 
+def setLevel(level):
+    logging.getLogger().setLevel(level)

getexp

@@ -10,15 +10,16 @@ import os
 import sys
 
 from expconfig import ExpConfig, ExpConfigError
-from feedback import die
-
-# Basic settings
-
+from feedback import setLevel, INFO, die, info
 
 #
 # Main routine
 #
 
+# Basic settings
+
+setLevel(INFO)
+
 # Check environment
 
 config_roots = os.environ.get('MKEXP_PATH', '').split(':')
@@ -32,6 +33,8 @@ command_line.add_argument('assigns', metavar='key=value', nargs='*',
                            help='override configuration file settings')
 command_line.add_argument('--path', '-p', 
                           help='search path for default config and templates')
+command_line.add_argument('--readme', '-R', action='store_true',
+                          help='only show header comment as in README file')
 command_line.add_argument('--verbose', '-v', action='store_true',
                           help='show additional information')
 
@@ -67,7 +70,9 @@ try:
 except ExpConfigError as error:
     die(error.message, status=2)
 
-if args.verbose:
+if args.readme:
+    print(config['EXP_DESCRIPTION'])
+elif args.verbose:
     items = config.items()
     items.sort(key=lambda x: x[0])
     for (key, value) in items:
@@ -84,5 +89,4 @@ if not (os.path.isdir(config['MODEL_DIR']) and
         os.path.isdir(config['SCRIPT_DIR']) and
         os.path.isdir(config['WORK_DIR']) and
         os.path.isdir(config['DATA_DIR'])):
-    die("data for experiment '{0}' does not exist".format(config.experiment_id))
-
+    info("data for experiment '{0}' does not exist".format(config.experiment_id))