diff --git a/Makefile b/Makefile index b2373b5327dec083049b005c93342483dee94411..dc4f04dbb4626a693f4c6cdf84e4ed3504515222 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ name = mkexp -version = 0.1.0 +version = 0.1.1dev package = $(name)-$(version) prefix = /usr/local diff --git a/diffexp b/diffexp index 92360e1d4109fb3a13f8a75978a5dd17e1a2302d..c5234c6ea3cc3d496c70ee80d63ffa1b2a33ca1c 100755 --- a/diffexp +++ b/diffexp @@ -13,16 +13,17 @@ die () { } [ "x$2" = x ] && die "Oops: invalid number of parameters -Usage: $PROGRAM config_a config_b" +Usage: $PROGRAM config_a config_b [key=value...]" CONFIG_A=$1 CONFIG_B=$2 +shift; shift -eval `getexp "$CONFIG_A" || echo \; exit $?` +eval `getexp "$CONFIG_A" "$@" || echo \; exit $?` EXP_A=$EXP_ID PATH_A=$SCRIPT_DIR -eval `getexp "$CONFIG_B" || echo \; exit $?` +eval `getexp "$CONFIG_B" "$@" || echo \; exit $?` EXP_B=$EXP_ID PATH_B=$SCRIPT_DIR diff --git a/examples/mean_values.config b/examples/mean_values.config index 242d7deea46839eb443918b3f06baf19471b9afe..e952e196a5da71234adedabf4e8663f52d68649f 100644 --- a/examples/mean_values.config +++ b/examples/mean_values.config @@ -1,5 +1,9 @@ # -# ECHAM experiment configuration file +# Performs a standard $EXP_TYPE simulation, but using monthy mean output. +# +# This experiment switches off the usual output and uses the mvstream module +# to create monthly mean data instead. The selection of variables reflects the +# standard selection that goes into the ATM, BOT and LOG files. # EXP_ID = mean_values diff --git a/examples/old_input_data.config b/examples/old_input_data.config index 2e21bebcb274c8518f449a0a9c95e3f49053bd3c..0c43bd5872aa7da654373f4171e9b94bb216bb31 100644 --- a/examples/old_input_data.config +++ b/examples/old_input_data.config @@ -1,5 +1,8 @@ # -# ECHAM experiment configuration file +# Performs a standard $EXP_TYPE simulation, using the original input data. +# +# Again, this is quite a simple setup, like reference, but for this experiment +# we use another input directory containing the original CMIP5 input data. # EXP_ID = old_input_data diff --git a/examples/reference.config b/examples/reference.config index 8e2eec524c64ad13e8da0a25daef8626bd30f990..e487c5446719e5ee27217e3dc80c9a31547f649c 100644 --- a/examples/reference.config +++ b/examples/reference.config @@ -1,5 +1,8 @@ # -# ECHAM experiment configuration file +# Performs a standard $EXP_TYPE simulation. +# +# This is an example for a very simple setup, changing only the model location, +# queue settings and accounting information. # EXP_ID = reference diff --git a/expconfig.py b/expconfig.py index 1f3e06d3c3759a643d76ed8a082f1a50ad215024..36257da4cfd0bf92717f4cfeca14b7414756cebd 100644 --- a/expconfig.py +++ b/expconfig.py @@ -8,6 +8,8 @@ import os import re import StringIO +from itertools import dropwhile + from configobj import ConfigObj, InterpolationError import feedback @@ -35,7 +37,7 @@ class ExpConfig(ConfigObj): # Class constructor - def __init__(self, experiment_config_name): + def __init__(self, experiment_config_name, extra_dict={}): '''Read experiment config to get basic settings TODO: probably nicer if default experiment is given as argument @@ -56,9 +58,7 @@ class ExpConfig(ConfigObj): pre_config = ConfigObj(experiment_config_name, interpolation=False) - self.experiment_id = pre_config['EXP_ID'] - experiment_type = pre_config['EXP_TYPE'] - self.experiment_kind = re.sub(r'-\wR$', '', experiment_type) + experiment_type = extra_dict.get('EXP_TYPE', pre_config['EXP_TYPE']) pre_config = None @@ -91,12 +91,16 @@ class ExpConfig(ConfigObj): # Re-read config to allow overriding default settings # TODO: probably nicer if default experiment is given as argument - pre_config.merge(ConfigObj(experiment_config_name, interpolation=False)) + experiment_config = ConfigObj(experiment_config_name, interpolation=False) + pre_config.merge(experiment_config) split_jobs(pre_config) # Add complete versioning info pre_config['VERSIONS_'] = ', '.join(config_versions) + # Add extra dictionary + pre_config.merge(extra_dict) + # Re-read merged config with interpolation set # This works around incomprehensible inheritance of interpolation with merge # Then make sure that all values are interpolated @@ -109,6 +113,22 @@ class ExpConfig(ConfigObj): config_lines.seek(0) ConfigObj.__init__(self, config_lines, interpolation='template') + if not self.has_key('EXP_DESCRIPTION'): + is_empty = lambda s: re.match(r'^\s*$', s) + rm_comment = lambda s: re.sub(r'^\s*# ?', '', s) + self['EXP_DESCRIPTION'] = "\n".join( + reversed(list( + dropwhile(is_empty, + reversed(list( + dropwhile(is_empty, + map(rm_comment, + experiment_config.initial_comment) + ) + )) + ) + )) + ) + def eval_key(section, key): try: section[key] = section[key] @@ -116,4 +136,6 @@ class ExpConfig(ConfigObj): raise ExpConfigError(error.message, key) self.walk(eval_key) + self.experiment_id = self['EXP_ID'] + self.experiment_kind = re.sub(r'-\wR$', '', experiment_type) diff --git a/feedback.py b/feedback.py index ae453b2343aa5ef4595f112030df3bbbf7f54939..c1f11add128a5a2a40aaebe6bb13ad8da04a99a3 100644 --- a/feedback.py +++ b/feedback.py @@ -14,6 +14,6 @@ warning = logging.warning error = logging.error critical = logging.critical -def die(message): +def die(message, status=1): error(message) - sys.exit(1) + sys.exit(status) diff --git a/getexp b/getexp index 6f5c8b7d93c54fa5157de060d25f3ce44176ff61..0a0a2b6bb7c258414045ed0a5ffe0dd7a01b87ad 100755 --- a/getexp +++ b/getexp @@ -17,21 +17,31 @@ from feedback import die # Check command line -if len(sys.argv) != 2: - die('invalid number of parameters\n' + - 'Usage: '+sys.argv[0]+' experiment_config_name') +program = sys.argv.pop(0) +usage = 'Usage: ' + program + ' experiment_config_name [key=value...]' + +if len(sys.argv) < 1: + feedback.die('invalid number of parameters\n' + usage) + +experiment_config_name = sys.argv.pop(0) -experiment_config_name = sys.argv[1] if not os.path.exists(experiment_config_name): - die("config file '{0}' does not exist".format(experiment_config_name)) + feedback.die("config file '{0}' does not exist".format(experiment_config_name)) + +invalid_args = filter(lambda x: not x.find('=')+1, sys.argv) + +if invalid_args: + feedback.die("invalid parameters ('"+"', '".join(invalid_args)+"')\n" + + usage) # Read and store configuration info from input and experiments' library # Store environment as default for control settings, then add config from files try: - config = ExpConfig(experiment_config_name) + config = ExpConfig(experiment_config_name, + dict(map(lambda x: x.split('=', 1), sys.argv))) except ExpConfigError as error: - die(error.message) + die(error.message, 2) print("EXP_ID='{0}'".format(config.experiment_id)) print("SCRIPT_DIR='{0}'".format(config['SCRIPT_DIR'])) diff --git a/mkexp b/mkexp index 6cd2e1988ceefe5f8b00087a3f225198ae60b9cc..fca26900195b42e76765b1d3e12e127efdce331d 100755 --- a/mkexp +++ b/mkexp @@ -119,24 +119,31 @@ def format_namelist(section): # Check command line -if len(sys.argv) < 2: - feedback.error('invalid number of parameters\n' + - 'Usage: '+sys.argv[0]+' experiment_config_name') - sys.exit(1) +program = sys.argv.pop(0) +usage = 'Usage: ' + program + ' experiment_config_name [key=value...]' + +if len(sys.argv) < 1: + feedback.die('invalid number of parameters\n' + usage) + +experiment_config_name = sys.argv.pop(0) -experiment_config_name = sys.argv[1] if not os.path.exists(experiment_config_name): - feedback.error("config file '{0}' does not exist".format(experiment_config_name)) - sys.exit(1) + feedback.die("config file '{0}' does not exist".format(experiment_config_name)) + +invalid_args = filter(lambda x: not x.find('=')+1, sys.argv) + +if invalid_args: + feedback.die("invalid parameters ('"+"', '".join(invalid_args)+"')\n" + + usage) # Read and store configuration info from input and experiments' library # Store environment as default for control settings, then add config from files try: - config = ExpConfig(experiment_config_name) + config = ExpConfig(experiment_config_name, + dict(map(lambda x: x.split('=', 1), sys.argv))) except ExpConfigError as error: - feedback.error(error.message) - sys.exit(2) + feedback.die(error.message, 2) # Create target directories @@ -183,3 +190,9 @@ for subjob, subconfig in config['jobs'].iteritems(): get_template_name(config.experiment_kind, template_job), get_script_name(config.experiment_id, subjob)) +# Create README file from experiment description + +readme_file = open(os.path.join(script_dir, 'README'), 'w') +readme_file.write(config['EXP_DESCRIPTION'] + '\n') +readme_file.close() + diff --git a/rmexp b/rmexp index 003f4741a3b88a1434542f4f576a96abfcc351c1..d10204edf36e1a7768133cee22996616647a26e6 100755 --- a/rmexp +++ b/rmexp @@ -13,13 +13,11 @@ die () { } [ "x$1" = x ] && die "Oops: invalid number of parameters -Usage: $PROGRAM config_file..." +Usage: $PROGRAM config_file [key=value...]" -for CONFIG -do +CONFIG=$1 +shift - eval `getexp "$CONFIG" || echo \; exit $?` +eval `getexp "$CONFIG" "$@" || echo \; exit $?` - rm -ri $SCRIPT_DIR $WORK_DIR - -done +rm -ri $SCRIPT_DIR $WORK_DIR