Skip to content
Snippets Groups Projects
Commit d518e0d1 authored by Karl-Hermann Wieners's avatar Karl-Hermann Wieners
Browse files

Release 0.1.0 of new script generation (second part, see also r2990)

* Changed directories to be fully configurable by default config
* Added tools for diff, printing, removal of experiment/script data
parent 52b94879
No related branches found
No related tags found
No related merge requests found
......@@ -11,9 +11,14 @@ mkexp
up directories, prepare input data, define configurations, and launch the
actual model run and processing.
expdiff
diffexp
is a tool that - given two mkexp configuration files - will compare the
generated scripts with either the standard diff program or any equivalent
tool, taking care of differences by experiment names etc. It is complemented
by expdiff_path which does the same kind of diff but takes its information
from the command line instead.
by diffpath which does the same kind of diff but takes its information from
the command line instead.
rmexp, getexp
More tools to remove experiment data, or get experiment info in
shell-readable form.
diffexp 0 → 100755
#! /bin/sh
#
# Compare experiment setups trying to identify file pairs.
#
PROGRAM=`basename $0`
BINDIR=`dirname $0`
PATH="$BINDIR:$PATH"
die () {
echo "$@" >&2
exit 1
}
[ "x$2" = x ] && die "Oops: invalid number of parameters
Usage: $PROGRAM config_a config_b"
CONFIG_A=$1
CONFIG_B=$2
eval `getexp "$CONFIG_A" || echo \; exit $?`
EXP_A=$EXP_ID
PATH_A=$SCRIPT_DIR
eval `getexp "$CONFIG_B" || echo \; exit $?`
EXP_B=$EXP_ID
PATH_B=$SCRIPT_DIR
exec diffpath "$EXP_A" "$EXP_B" "$PATH_A" "$PATH_B"
diffpath 0 → 100755
#! /bin/sh
#
# Compare experiment setups trying to identify file pairs.
#
PROGRAM=`basename $0`
die () {
echo "$@" >&2
exit 1
}
DIFF=${DIFF:-diff}
diff_exp_files () (
exp_a=$1
exp_b=$2
file_a=$3
file_b=$4
if [ -e "$file_b" ]
then
trap 'rm -f $temp_b' 0
temp_b=`mktemp`
if [ -r "$file_b" ]
then
sed "s,$exp_b,$exp_a,g" "$file_b" > $temp_b
else
chmod 0000 $temp_b
fi
else
temp_b=$file_b
fi
$DIFF --label "$file_a" --label "$file_b" "$file_a" "$temp_b"
)
[ "x$3" = x ] && die "Oops: invalid number of parameters
Usage: $PROGRAM experiment_id_a experiment_id_b path_to_a [path_to_b]"
EXP_A=$1
EXP_B=$2
PATH_A=$3
# If path to b is not set or empty, use path to a as template
PATH_B=${4:-`echo "$PATH_A" | sed "s,$EXP_A,$EXP_B,g"`}
trap 'rm -r $DIFF_FILE' 0
DIFF_FILE=`mktemp`
for FILE_A in $(
{
(cd "$PATH_A" && find . ! -type d ! -name '*.log')
(cd "$PATH_B" && find . ! -type d ! -name '*.log' | sed "s,$EXP_B,$EXP_A,g")
} | sed 's,^\./,,' | sort -u
)
do
FILE_B=`echo "$FILE_A" | sed "s,$EXP_A,$EXP_B,g"`
diff_exp_files "$EXP_A" "$EXP_B" "$PATH_A/$FILE_A" "$PATH_B/$FILE_B" > $DIFF_FILE ||
STATUS=1
sed "1i Index: $FILE_A\n\
===================================================================" $DIFF_FILE
done
exit $STATUS
......@@ -8,9 +8,7 @@ EXP_TYPE = amip-LR
QUEUE_TYPE = blizzard
ACCOUNT = xy9999
model = prj/issues/02114/echam-dev
MODEL_ROOT = $HOME/$model
DATA_ROOT = $SCRATCH/$model # SCRATCH is defined by blizzard login scripts
MODEL_SUBDIR = prj/issues/02114/echam-dev
INITIAL_DATA = /pool/data/ECHAM6/input/r0001
......@@ -8,7 +8,5 @@ EXP_TYPE = amip-LR
QUEUE_TYPE = blizzard
ACCOUNT = xy9999
model = prj/issues/02114/echam-dev
MODEL_SUBDIR = prj/issues/02114/echam-dev
MODEL_ROOT = $HOME/$model
DATA_ROOT = $SCRATCH/$model # SCRATCH is defined by blizzard login scripts
#
# Generate an earth system model configuration from the given configuration file.
#
# $Id$
#
import os
import re
import StringIO
from configobj import ConfigObj, InterpolationError
import feedback
class ExpConfigError(InterpolationError):
def __init__(self, message, key):
InterpolationError.__init__(self,
"{0} while reading key '{1}'".format(message[:-1], key))
class ExpConfig(ConfigObj):
'''Read and store configuration info from input and experiments' library
Store environment as default for control settings, then add config from files
'''
#
# Basic settings
#
exp_lib_dir = 'standard_experiments'
default_name = 'DEFAULT'
# Config post-processing
# Class constructor
def __init__(self, experiment_config_name):
'''Read experiment config to get basic settings
TODO: probably nicer if default experiment is given as argument
'''
def split_jobs(config):
'''Post-process job definition to allow for shared configs as [[job1, job2]]'''
if 'jobs' in config:
sep = re.compile(r'\s*,\s')
for subjobs, subconfig in config['jobs'].iteritems():
if re.search(sep, subjobs):
for subjob in re.split(sep, subjobs):
if subjob in config['jobs']:
config['jobs'][subjob].merge(subconfig.dict())
else:
config['jobs'][subjob] = subconfig.dict()
del config['jobs'][subjobs]
pre_config = ConfigObj(experiment_config_name, interpolation=False)
self.experiment_id = pre_config['EXP_ID']
experiment_type = pre_config['EXP_TYPE']
self.experiment_kind = re.sub(r'-\wR$', '', experiment_type)
pre_config = None
# Start from empty configuration
pre_config = ConfigObj(interpolation=False)
config_versions = []
# Read Environment
pre_config.merge({'DEFAULT': dict(os.environ)})
# Read settings from library (general default and experiment type specific)
lib_config_name = os.path.join(ExpConfig.exp_lib_dir,
ExpConfig.default_name+'.config')
pre_config.merge(ConfigObj(lib_config_name, interpolation=False))
split_jobs(pre_config)
config_versions.append(pre_config['VERSION_'])
del pre_config['VERSION_']
lib_config_name = os.path.join(ExpConfig.exp_lib_dir,
experiment_type+'.config')
if os.path.exists(lib_config_name):
pre_config.merge(ConfigObj(lib_config_name, interpolation=False))
split_jobs(pre_config)
config_versions.append(pre_config['VERSION_'])
del pre_config['VERSION_']
else:
feedback.warning("cannot find config for '%s', using default only",
experiment_type)
# Re-read config to allow overriding default settings
# TODO: probably nicer if default experiment is given as argument
pre_config.merge(ConfigObj(experiment_config_name, interpolation=False))
split_jobs(pre_config)
# Add complete versioning info
pre_config['VERSIONS_'] = ', '.join(config_versions)
# Re-read merged config with interpolation set
# This works around incomprehensible inheritance of interpolation with merge
# Then make sure that all values are interpolated
config_lines = StringIO.StringIO()
pre_config.write(config_lines)
pre_config = None
config_lines.seek(0)
ConfigObj.__init__(self, config_lines, interpolation='template')
def eval_key(section, key):
try:
section[key] = section[key]
except InterpolationError as error:
raise ExpConfigError(error.message, key)
self.walk(eval_key)
'''Frontend for using logging module as terminal feedback facility'''
import logging
import sys
logging.basicConfig(format='%(levelname)s: %(message)s')
logging.addLevelName(logging.INFO, 'Note')
logging.addLevelName(logging.WARNING, 'Hey')
logging.addLevelName(logging.ERROR, 'Oops')
logging.addLevelName(logging.CRITICAL, 'Sorry')
info = logging.info
warning = logging.warning
error = logging.error
critical = logging.critical
def die(message):
error(message)
sys.exit(1)
getexp 0 → 100755
#! /usr/bin/env python
#
# Print experiment info in shell-digestible form
#
import os
import sys
from expconfig import ExpConfig, ExpConfigError
from feedback import die
# Utilities
#
# Main routine
#
# Check command line
if len(sys.argv) != 2:
die('invalid number of parameters\n' +
'Usage: '+sys.argv[0]+' experiment_config_name')
experiment_config_name = sys.argv[1]
if not os.path.exists(experiment_config_name):
die("config file '{0}' does not exist".format(experiment_config_name))
# Read and store configuration info from input and experiments' library
# Store environment as default for control settings, then add config from files
try:
config = ExpConfig(experiment_config_name)
except ExpConfigError as error:
die(error.message)
print("EXP_ID='{0}'".format(config.experiment_id))
print("SCRIPT_DIR='{0}'".format(config['SCRIPT_DIR']))
print("WORK_DIR='{0}'".format(config['WORK_DIR']))
print("DATA_DIR='{0}'".format(config['DATA_DIR']))
if not (os.path.isdir(config['SCRIPT_DIR']) and
os.path.isdir(config['WORK_DIR']) and
os.path.isdir(config['DATA_DIR'])):
die("data for experiment '{0}' does not exist".format(config.experiment_id))
......@@ -5,30 +5,21 @@
# $Id$
#
import logging
import os
import re
import stat
import StringIO
import sys
from configobj import ConfigObj, InterpolationError
from jinja2 import Environment, FileSystemLoader
from expconfig import ExpConfig, ExpConfigError
import feedback
#
# Basic settings
#
exp_lib_dir = 'standard_experiments'
que_lib_dir = 'standard_queue_settings'
default_name = 'DEFAULT'
logging.basicConfig(format='%(levelname)s: %(message)s')
logging.addLevelName(logging.INFO, 'Note')
logging.addLevelName(logging.WARNING, 'Hey')
logging.addLevelName(logging.ERROR, 'Oops')
logging.addLevelName(logging.CRITICAL, 'Sorry')
# Setup templating environment
template_env = Environment(
loader = FileSystemLoader('.'),
......@@ -59,10 +50,12 @@ def chmod_plus_x(file_name):
def get_template_name(experiment_kind, job_id):
'''Determine script template, taking system default if type is unsupported.'''
run_template_name = os.path.join(exp_lib_dir, experiment_kind+'.'+job_id+'.tmpl')
run_template_name = os.path.join(ExpConfig.exp_lib_dir,
experiment_kind+'.'+job_id+'.tmpl')
if not os.path.exists(run_template_name):
logging.info("no %s template for '%s', using default", job_id, experiment_kind)
run_template_name = os.path.join(exp_lib_dir, default_name+'.'+job_id+'.tmpl')
feedback.info("no %s template for '%s', using default", job_id, experiment_kind)
run_template_name = os.path.join(ExpConfig.exp_lib_dir,
ExpConfig.default_name+'.'+job_id+'.tmpl')
return run_template_name
def get_script_name(experiment_id, job_id):
......@@ -120,21 +113,6 @@ def format_namelist(section):
lines.write('/\n')
return lines.getvalue()
# Config post-processing
def split_jobs(config):
'''Post-process job definition to allow for shared configs as [[job1, job2]]'''
if 'jobs' in config:
sep = re.compile(r'\s*,\s')
for subjobs, subconfig in config['jobs'].iteritems():
if re.search(sep, subjobs):
for subjob in re.split(sep, subjobs):
if subjob in config['jobs']:
config['jobs'][subjob].merge(subconfig.dict())
else:
config['jobs'][subjob] = subconfig.dict()
del config['jobs'][subjobs]
#
# Main routine
#
......@@ -142,104 +120,50 @@ def split_jobs(config):
# Check command line
if len(sys.argv) < 2:
logging.error('invalid number of parameters\n' +
feedback.error('invalid number of parameters\n' +
'Usage: '+sys.argv[0]+' experiment_config_name')
sys.exit(1)
experiment_config_name = sys.argv[1]
if not os.path.exists(experiment_config_name):
logging.error("config file '{0}' does not exist".format(experiment_config_name))
feedback.error("config file '{0}' does not exist".format(experiment_config_name))
sys.exit(1)
# Read and store configuration info from input and experiments' library
# Store environment as default for control settings, then add config from files
# Read experiment config to get basic settings
# TODO: probably nicer if default experiment is given as argument
pre_config = ConfigObj(experiment_config_name, interpolation=False)
experiment_id = pre_config['EXP_ID']
experiment_type = pre_config['EXP_TYPE']
experiment_kind = re.sub(r'-\wR$', '', experiment_type)
queue_type = pre_config['QUEUE_TYPE']
pre_config = None
# Start from empty configuration
pre_config = ConfigObj(interpolation=False)
config_versions = []
# Read Environment
pre_config.merge({'DEFAULT': dict(os.environ)})
# Read settings from library (general default and experiment type specific)
lib_config_name = os.path.join(exp_lib_dir, default_name+'.config')
pre_config.merge(ConfigObj(lib_config_name, interpolation=False))
split_jobs(pre_config)
config_versions.append(pre_config['VERSION_'])
del pre_config['VERSION_']
lib_config_name = os.path.join(exp_lib_dir, experiment_type+'.config')
if os.path.exists(lib_config_name):
pre_config.merge(ConfigObj(lib_config_name, interpolation=False))
split_jobs(pre_config)
config_versions.append(pre_config['VERSION_'])
del pre_config['VERSION_']
else:
logging.warning("cannot find config for '%s', using default only", experiment_type)
# Re-read config to allow overriding default settings
# TODO: probably nicer if default experiment is given as argument
pre_config.merge(ConfigObj(experiment_config_name, interpolation=False))
split_jobs(pre_config)
# Add complete versioning info
pre_config['VERSIONS_'] = ', '.join(config_versions)
# Re-read merged config with interpolation set
# This works around incomprehensible inheritance of interpolation with merge
# Then make sure that all values are interpolated
config_lines = StringIO.StringIO()
pre_config.write(config_lines)
pre_config = None
config_lines.seek(0)
config = ConfigObj(config_lines, interpolation='template')
def eval_key(section, key):
try:
section[key] = section[key]
except InterpolationError as error:
logging.error("{0} while reading key '{1}'".format(error.message[:-1], key))
sys.exit(2)
config.walk(eval_key)
try:
config = ExpConfig(experiment_config_name)
except ExpConfigError as error:
feedback.error(error.message)
sys.exit(2)
# Create target directories
# Create directory for scripts if it doesn't exist
script_dir = os.path.join(config['MODEL_ROOT'],
'experiments',
experiment_id,
'scripts')
script_dir = config['SCRIPT_DIR']
print "Script directory: '"+script_dir+"'"
if not os.path.isdir(script_dir):
os.makedirs(script_dir)
else:
logging.warning("script directory already exists, overwriting existing scripts")
feedback.warning("script directory already exists, overwriting existing scripts")
# Create directory for output data if it doesn't exist
data_dir = os.path.join(config['DATA_ROOT'],
'experiments',
experiment_id)
data_dir = config['DATA_DIR']
print "Data directory: '"+data_dir+"'"
if not os.path.isdir(data_dir):
os.makedirs(data_dir)
else:
logging.warning("data directory already exists")
feedback.warning("data directory already exists")
# Create directory for running the experiment if it doesn't exist
work_dir = config['WORK_DIR']
if work_dir != data_dir:
print "Work directory: '"+work_dir+"'"
if not os.path.isdir(work_dir):
os.makedirs(work_dir)
else:
feedback.warning("work directory already exists")
# Prepare namelists for inclusion in scripts
for namelist, groups in config['namelists'].iteritems():
......@@ -256,6 +180,6 @@ for subjob, subconfig in config['jobs'].iteritems():
if 'extends' in config['JOB']:
template_job = config['JOB']['extends']
expand_template(config,
get_template_name(experiment_kind, template_job),
get_script_name(experiment_id, subjob))
get_template_name(config.experiment_kind, template_job),
get_script_name(config.experiment_id, subjob))
rmexp 0 → 100755
#! /bin/sh
#
# Compare experiment setups trying to identify file pairs.
#
PROGRAM=`basename $0`
BINDIR=`dirname $0`
PATH="$BINDIR:$PATH"
die () {
echo "$@" >&2
exit 1
}
[ "x$1" = x ] && die "Oops: invalid number of parameters
Usage: $PROGRAM config_file..."
for CONFIG
do
eval `getexp "$CONFIG" || echo \; exit $?`
rm -ri $SCRIPT_DIR $WORK_DIR
done
from distutils.core import setup
name = 'mkexp'
version = '0.1.0dev'
version = '0.1.0'
setup(
name = name,
......@@ -11,11 +11,12 @@ setup(
author = 'Karl-Hermann Wieners',
author_email = 'karl-hermann.wieners@mpimet.mpg.de',
url = 'http://code.zmaw.de/projects/esmenv',
py_modules = ['configobj', 'validate'],
scripts = ['mkexp'],
py_modules = ['configobj', 'validate', 'feedback', 'expconfig'],
scripts = ['mkexp', 'getexp', 'rmexp', 'diffexp', 'diffpath'],
platforms = ['Posix'],
license = 'LICENSE.txt',
data_files = [('share/doc/'+name+'-'+version+'/examples',
['examples/khw0000.config'])],
['examples/'+example+'.config'
for example in ['reference', 'old_input_data', 'mean_values']])],
requires = ['Jinja2(>= 2.6)']
)
\ No newline at end of file
)
......@@ -5,8 +5,25 @@
VERSION_ = $$Id$$
ECHAM_EXE = echam6
INITIAL_DATA = /pool/data/ECHAM6/input/r0002
MODEL_ROOT = $HOME
SCRIPT_ROOT = $MODEL_ROOT
DATA_ROOT = /work/$ACCOUNT/$LOGNAME
WORK_ROOT = $DATA_ROOT
MODEL_SUBDIR = echam-dev
EXPERIMENTS_SUBDIR = experiments
SCRIPTS_SUBDIR = scripts
DATA_SUBDIR =
WORK_SUBDIR =
MODEL_DIR = $MODEL_ROOT/$MODEL_SUBDIR
SCRIPT_DIR = $SCRIPT_ROOT/$MODEL_SUBDIR/$EXPERIMENTS_SUBDIR/$EXP_ID/$SCRIPTS_SUBDIR
DATA_DIR = $DATA_ROOT/$MODEL_SUBDIR/$EXPERIMENTS_SUBDIR/$EXP_ID/$DATA_SUBDIR
WORK_DIR = $WORK_ROOT/$MODEL_SUBDIR/$EXPERIMENTS_SUBDIR/$EXP_ID/$WORK_SUBDIR
[jobs]
[[run, run_start]]
......
......@@ -13,11 +13,9 @@ set -ex
#MONTH
#YEAR
###############################################################################
export expid=%{EXP_ID}
export LEVELS=%{LEVELS}
#
export jobdir=%{MODEL_ROOT}/experiments/$expid
export data=%{DATA_ROOT}/experiments/$expid
export data=%{DATA_DIR}
#
export cdo=/client/bin/cdo
export after=/sw/aix61/after-4.6.2/bin/after
......
......@@ -36,8 +36,6 @@ START=%{JOB.start}
EXP="%{EXP_ID}" # experiment identifier
expname="%{EXP_TYPE}" # experiments name
#
RERUN=.true. # Rerun switch; .false. for initial run, .true. else
#
NTHREADS=%{JOB.threads_per_task}
export ECHAM6_THREADS=$NTHREADS
#
......@@ -47,13 +45,11 @@ OCERES=%{OCERES}
#
#-----------------------------------------------------------------------------
#
WORK_DIR=%{MODEL_ROOT}
#
# absolute path to directory with job scripts:
SCRIPTDIR=$WORK_DIR/experiments/$EXP/scripts
SCRIPTDIR=%{SCRIPT_DIR}
#
# absolute path to directory with plenty of space:
EXPDIR=%{DATA_ROOT}/experiments/$EXP
EXPDIR=%{WORK_DIR}
#
# absolute path to directory with initial data:
INI_DATA=%{INITIAL_DATA}
......@@ -68,7 +64,7 @@ INIJSBTRANS=$INIJSB/New_Hampshire_LCC
LIBJSB=/pool/data/ECHAM6/jsbach
#
# absolute path to model binary, including the executable
MODEL=${WORK_DIR}/bin/%{ECHAM_EXE}
MODEL=%{MODEL_DIR}/bin/%{ECHAM_EXE}
#
#-----------------------------------------------------------------------------
#
......
......@@ -34,9 +34,6 @@ START=%{JOB.start}
# followed by a number for the current experiment
#
EXP="%{EXP_ID}" # experiment identifier
expname="%{EXP_TYPE}" # experiments name
#
RERUN=.true. # Rerun switch; .false. for initial run, .true. else
#
NTHREADS=%{JOB.threads_per_task}
export ECHAM6_THREADS=$NTHREADS
......@@ -47,13 +44,11 @@ OCERES=%{OCERES}
#
#-----------------------------------------------------------------------------
#
WORK_DIR=%{MODEL_ROOT}
#
# absolute path to directory with job scripts:
SCRIPTDIR=$WORK_DIR/experiments/$EXP/scripts
SCRIPTDIR=%{SCRIPT_DIR}
#
# absolute path to directory with plenty of space:
EXPDIR=%{DATA_ROOT}/experiments/$EXP
EXPDIR=%{WORK_DIR}
#
# absolute path to directory with initial data:
INI_DATA=%{INITIAL_DATA}
......@@ -68,7 +63,7 @@ INIJSBTRANS=$INIJSB/New_Hampshire_LCC
LIBJSB=/pool/data/ECHAM6/jsbach
#
# absolute path to model binary, including the executable
MODEL=${WORK_DIR}/bin/%{ECHAM_EXE}
MODEL=%{MODEL_DIR}/bin/%{ECHAM_EXE}
#
#-----------------------------------------------------------------------------
#
......
......@@ -34,9 +34,6 @@ START=%{JOB.start}
# followed by a number for the current experiment
#
EXP="%{EXP_ID}" # experiment identifier
expname="%{EXP_TYPE}" # experiments name
#
RERUN=.true. # Rerun switch; .false. for initial run, .true. else
#
NTHREADS=%{JOB.threads_per_task}
export ECHAM6_THREADS=$NTHREADS
......@@ -47,13 +44,11 @@ OCERES=%{OCERES}
#
#-----------------------------------------------------------------------------
#
WORK_DIR=%{MODEL_ROOT}
#
# absolute path to directory with job scripts:
SCRIPTDIR=$WORK_DIR/experiments/$EXP/scripts
SCRIPTDIR=%{SCRIPT_DIR}
#
# absolute path to directory with plenty of space:
EXPDIR=%{DATA_ROOT}/experiments/$EXP
EXPDIR=%{WORK_DIR}
#
# absolute path to directory with initial data:
INI_DATA=%{INITIAL_DATA}
......@@ -68,7 +63,7 @@ INIJSBTRANS=$INIJSB/New_Hampshire_LCC
LIBJSB=/pool/data/ECHAM6/jsbach
#
# absolute path to model binary, including the executable
MODEL=${WORK_DIR}/bin/%{ECHAM_EXE}
MODEL=%{MODEL_DIR}/bin/%{ECHAM_EXE}
#
#-----------------------------------------------------------------------------
#
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment