diff --git a/CHANGES.txt b/CHANGES.txt
index 7c081329c77e3c3edf509c97efc8ae1583a305e2..d39d7c77ce5abfe4cca7bf90015beffec1d7da22 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,13 +6,13 @@ Make Experiments!
Release Changes
---------------
-Milestone 1.1.1
-===============
+Release 1.1.1
+=============
Global
------
-* TODO: Porting to python3
+* Updated code to run with python3 and full Unicode set
Configuration
-------------
diff --git a/Makefile b/Makefile
index a0df22993833a4903b9af775552fd38d53567847..d58d87f44592cdac3832c0448632ac1acce7de4e 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ distclean: clean
doc: doc/mkexp.pdf
check:
- $(PYTHON) test.py -v
+ PYTHON=$(PYTHON) $(PYTHON) test.py -v
dist: doc
$(PYTHON) setup.py sdist
diff --git a/doc/mkexp.fodt b/doc/mkexp.fodt
index 442e625b319009e93fc063efcd1ae2aae9878f4d..56ab80a9895d76d7202f255f17ae4e881b3ef946 100644
--- a/doc/mkexp.fodt
+++ b/doc/mkexp.fodt
@@ -2708,7 +2708,7 @@
</text:sequence-decls>
<text:p text:style-name="P349">Make Experiments!</text:p>
<text:p text:style-name="P338">Run-script generation for earth system models</text:p>
- <text:p text:style-name="P339">Release 1.1.1rc2</text:p>
+ <text:p text:style-name="P339">Release 1.1.1</text:p>
<text:p text:style-name="P312">Karl-Hermann Wieners<text:line-break/><text:span text:style-name="T274">Max-Planck-Institut für Meteorologie<text:line-break/>Hamburg</text:span></text:p>
<text:table-of-content text:style-name="Sect1" text:protected="true" text:name="Table of Contents1">
<text:table-of-content-source text:outline-level="10">
@@ -3461,4 +3461,4 @@
<text:p text:style-name="P47"><text:span text:style-name="T140">This</text:span><text:span text:style-name="T148"> will result in a '.run_first' </text:span><text:span text:style-name="T149">script</text:span><text:span text:style-name="T148"> tha</text:span><text:span text:style-name="T149">t gets the model state from 'anotherexp', while the '.run' script</text:span><text:span text:style-name="T148"> </text:span><text:span text:style-name="T149">uses 'myexp'.</text:span></text:p>
</office:text>
</office:body>
-</office:document>
\ No newline at end of file
+</office:document>
diff --git a/doc/mkexp.pdf b/doc/mkexp.pdf
index a836f8f9afab24a22b0be11c38299bcb0eb70d5c..a126a268b522063dc6931389b2c0cc6bd0fed16a 100644
Binary files a/doc/mkexp.pdf and b/doc/mkexp.pdf differ
diff --git a/expargparse.py b/expargparse.py
index dc49f87c075fd0be784d67ebb3f672537211e7c3..a73c05132e5f45e69a6eedb28f4d1ea835e3e1cd 100644
--- a/expargparse.py
+++ b/expargparse.py
@@ -63,5 +63,5 @@ def assigns_to_dicts(args):
current = {key: current}
return current
- return map(assign_to_dict, args.assigns)
+ return list(map(assign_to_dict, args.assigns))
diff --git a/expconfig.py b/expconfig.py
index e12156c6a87b2f68c3a012d219d4be837e6e5e5f..0c34c31c7ebe2a36ff9fc6d563273e1a795f6e08 100644
--- a/expconfig.py
+++ b/expconfig.py
@@ -4,9 +4,10 @@ Generate an earth system model configuration from the given configuration file.
$Id$
'''
+import io
+import locale
import os
import re
-import StringIO
import time # for 'eval' context only
from itertools import dropwhile
@@ -18,16 +19,24 @@ import feedback
# Utilities used for config and templates
+_preferred_encoding = locale.getpreferredencoding()
+
# - Check a namelist logical
def is_set(s):
if not s:
return False
return s.strip('.').lower().startswith('t')
-
class ConfigObj(configobj.ConfigObj):
def __init__(self, *args, **kwargs):
+ default_args = {
+ 'encoding': _preferred_encoding,
+ 'default_encoding': _preferred_encoding,
+ }
+ for kw in default_args:
+ if not kw in kwargs:
+ kwargs[kw] = default_args[kw]
configobj.ConfigObj.__init__(self, *args, **kwargs)
def merge(self, indict):
@@ -35,10 +44,10 @@ class ConfigObj(configobj.ConfigObj):
def is_not_empty(arg):
if arg is None:
return None
- elif isinstance(arg, basestring):
+ elif not isinstance(arg, (list, tuple)):
return arg.rstrip()
else:
- return filter(None, map(lambda x: x.rstrip(), arg))
+ return [_f for _f in [x.rstrip() for x in arg] if _f]
def merge_comments(this, indict):
'''Merge comments from indict into current configuration.
@@ -108,7 +117,7 @@ class ExpConfig(ConfigObj):
'''Post-process job definition to allow for shared configs as [[job1, job2]]'''
if 'jobs' in config:
sep = re.compile(r'\s*,\s*')
- for subjobs, subconfig in config['jobs'].iteritems():
+ for subjobs, subconfig in config['jobs'].items():
if re.search(sep, subjobs):
for subjob in re.split(sep, subjobs):
if subjob in config['jobs']:
@@ -164,7 +173,7 @@ class ExpConfig(ConfigObj):
def add_years(value, years):
'''Add specified number of years (possible negative) to date'''
years = int(years)
- dt = map(int, split_date(value))
+ dt = list(map(int, split_date(value)))
dt[0] += years
return "{0:+05}-{1:02}-{2:02}".format(*dt).lstrip('+')
@@ -203,7 +212,7 @@ class ExpConfig(ConfigObj):
return (year, mon, day)
days = int(days)
- dt = map(int, split_date(value))
+ dt = list(map(int, split_date(value)))
dt = add_days_(dt[0], dt[1], dt[2], days)
return "{0:+05}-{1:02}-{2:02}".format(*dt).lstrip('+')
@@ -214,7 +223,7 @@ class ExpConfig(ConfigObj):
'''
result = eval(value)
if isinstance(result, (list, tuple)):
- result = map(str, result)
+ result = list(map(str, result))
else:
result = str(result)
return result
@@ -267,12 +276,12 @@ class ExpConfig(ConfigObj):
try:
value = section[key]
if isinstance(value, (list, tuple)):
- value = map(eval_expression, value)
- elif isinstance(value, basestring):
+ value = list(map(eval_expression, value))
+ elif not isinstance(value, dict):
value = eval_expression(value)
if isinstance(value, (list, tuple)):
value = [v.replace('$', '$$') for v in value]
- elif isinstance(value, basestring):
+ elif not isinstance(value, dict):
value = value.replace('$', '$$')
except (InterpolationError, ValueError) as error:
raise ExpConfigError(error.message, key)
@@ -284,7 +293,7 @@ class ExpConfig(ConfigObj):
value = section[key]
if isinstance(value, (list, tuple)):
value = [v.replace('$$', '$') for v in value]
- elif isinstance(value, basestring):
+ elif not isinstance(value, dict):
value = value.replace('$$', '$')
except (InterpolationError, ValueError) as error:
raise ExpConfigError(error.message, key)
@@ -323,7 +332,7 @@ class ExpConfig(ConfigObj):
setup_options = extra_dict.get('SETUP_OPTIONS',
pre_config.get('SETUP_OPTIONS',
''))
- if isinstance(setup_options, basestring):
+ if not isinstance(setup_options, (list, tuple)):
if setup_options:
setup_options = [setup_options]
else:
@@ -331,7 +340,7 @@ class ExpConfig(ConfigObj):
exp_options = extra_dict.get('EXP_OPTIONS',
pre_config.get('EXP_OPTIONS',
''))
- if isinstance(exp_options, basestring):
+ if not isinstance(exp_options, (list, tuple)):
if exp_options:
exp_options = [exp_options]
else:
@@ -364,10 +373,10 @@ class ExpConfig(ConfigObj):
env_dict = dict(os.environ)
if not getexp:
# Mask literal dollar characters
- for key, value in env_dict.iteritems():
+ for key, value in env_dict.items():
env_dict[key] = value.replace('$', '$$')
pre_config.merge({'DEFAULT': {}})
- for key, value in sorted(env_dict.iteritems()):
+ for key, value in sorted(env_dict.items()):
pre_config['DEFAULT'][key] = value
# Read experiment settings from library (default and type specific)
@@ -438,18 +447,18 @@ class ExpConfig(ConfigObj):
# This works around incomprehensible inheritance of interpolation with
# merge. Make sure that all values are interpolated
- config_lines = StringIO.StringIO()
+ config_lines = io.BytesIO()
pre_config.write(config_lines)
pre_config = None
config_lines.seek(0)
- pre_config = ConfigObj(config_lines,
+ pre_config = ConfigObj(io.TextIOWrapper(config_lines),
interpolation=False if getexp else 'template')
# Extract experiment description from initial comment
# if not set explicitly
- if not pre_config.has_key('EXP_DESCRIPTION'):
+ if 'EXP_DESCRIPTION' not in pre_config:
is_empty = lambda s: re.match(r'^[\s#]*$', s)
rm_comment = lambda s: re.sub(r'^\s*# ?', '', s)
pre_config['EXP_DESCRIPTION'] = "\n".join(
@@ -457,8 +466,8 @@ class ExpConfig(ConfigObj):
dropwhile(is_empty,
reversed(list(
dropwhile(is_empty,
- map(rm_comment,
- experiment_config.initial_comment)
+ list(map(rm_comment,
+ experiment_config.initial_comment))
)
))
)
@@ -470,14 +479,14 @@ class ExpConfig(ConfigObj):
# Re-read final config without interpolation.
# This allows copying data without evaluation of version keywords.
- config_lines.seek(0)
- config_lines.truncate()
-
+ config_lines = io.BytesIO()
+
pre_config.write(config_lines)
pre_config = None
config_lines.seek(0)
- ConfigObj.__init__(self, config_lines, interpolation=False)
+ ConfigObj.__init__(self, io.TextIOWrapper(config_lines),
+ interpolation=False)
self.walk(uneval_key)
self.experiment_id = self[ExpConfig.id_name]
diff --git a/getconfig b/getconfig
index b964230e5135fb23c75d53482c93ccf59a21a834..4b67f900eb4fc16f13961cedeeb2b557b65c2fd1 100755
--- a/getconfig
+++ b/getconfig
@@ -5,10 +5,13 @@
# $Id$
#
+from __future__ import print_function
+
import argparse
+import io
import sys
-from configobj import ConfigObj
+from expconfig import ConfigObj
import update
from feedback import die
@@ -39,5 +42,12 @@ config_data = ConfigObj(update_data.get_config_file(), interpolation=False,
for d in update_data.get_config_dicts():
config_data.merge(d)
-config_data.write(sys.stdout)
+# Ready to roll out
+
+lines = io.BytesIO()
+config_data.write(lines)
+
+lines.seek(0)
+for line in io.TextIOWrapper(lines):
+ print(line, end='')
diff --git a/getexp b/getexp
index c704fa1c2a73f676508f161d4faf368a539d59a9..18ba8877aae14e35a424790e5a5835ba47081ef2 100755
--- a/getexp
+++ b/getexp
@@ -5,7 +5,10 @@
# $Id$
#
+from __future__ import print_function
+
import argparse
+import io
import os
import sys
@@ -72,7 +75,7 @@ if not os.path.exists(experiment_config_name):
die("config file '{0}' does not exist".format(experiment_config_name))
# Overrides
-invalid_args = filter(lambda x: not x.find('=')+1, args.assigns)
+invalid_args = [x for x in args.assigns if not x.find('=')+1]
if invalid_args:
die("invalid parameters ('"+"', '".join(invalid_args)+"')\n" +
@@ -82,12 +85,12 @@ if invalid_args:
# Store environment as default for control settings, then add config from files
# Hack to allow use in diffexp
-if os.environ.has_key('DIFF'):
+if 'DIFF' in os.environ:
del os.environ['DIFF']
try:
config = ExpConfig(experiment_config_name,
- dict(map(lambda x: x.split('=', 1), args.assigns)),
+ dict([x.split('=', 1) for x in args.assigns]),
config_roots)
except ExpConfigError as error:
die(error.message, status=2)
@@ -105,13 +108,17 @@ elif args.key:
die("invalid config name '{0}'".format(error.message))
elif args.verbose >= 2:
config.indent_type = ' '
- config.write(sys.stdout)
+ lines = io.BytesIO()
+ config.write(lines)
+ lines.seek(0)
+ for line in io.TextIOWrapper(lines):
+ print(line, end='')
elif args.verbose == 1:
- items = config.items()
- items.sort(key=lambda x: x[0])
- for (key, value) in items:
- if not isinstance(value, dict):
- print("{0}='{1}'".format(key, str(value).replace("'",'"')))
+ for key in sorted(config.scalars):
+ value = config[key]
+ if isinstance(value, (list, tuple)):
+ value = u'"{0}"'.format(u'" "'.join(value))
+ print(u"{0}='{1}'".format(key, value.replace("'",'"')))
else:
print("EXP_ID='{0}'".format(config.experiment_id))
print("MODEL_DIR='{0}'".format(config['MODEL_DIR']))
diff --git a/mkexp b/mkexp
index 18740082caabc83ee5a179f0596f699763eb942c..de30beeffa8b88d8b1e506eff72e2270234e60c0 100755
--- a/mkexp
+++ b/mkexp
@@ -5,24 +5,22 @@
# $Id$
#
-import codecs
import copy
+import io
import os
import re
import stat
-import StringIO
import sys
import textwrap
from time import strftime
-from configobj import ConfigObj
import jinja2
from jinja2 import Environment, ChoiceLoader, FileSystemLoader, \
TemplateNotFound, TemplatesNotFound, is_undefined
import expargparse
import expconfig
-from expconfig import ExpConfig, ExpConfigError
+from expconfig import ConfigObj, ExpConfig, ExpConfigError
import feedback
import files
import package_info
@@ -37,9 +35,6 @@ import package_info
# File system
-def open_utf8(name, mode='r'):
- return codecs.open(name, mode, encoding='utf8')
-
def chmod_plus_x(file_name):
'''Make a file executable, respecting user mask.'''
# Get umask
@@ -79,13 +74,13 @@ def expand_template(template_dict, template_name):
def expand_template_file(template_dict, template_names, expanded_name, backup_name):
'''Replace keywords in template file using the given dictionary.'''
move_file_to_backup(expanded_name, backup_name)
- expanded_file = open_utf8(expanded_name, 'w')
+ expanded_file = io.open(expanded_name, 'w')
try:
for line in template_env.select_template(template_names).generate(template_dict):
expanded_file.write(line)
except TemplatesNotFound as error:
feedback.die(error.message)
- expanded_file.write('\n')
+ expanded_file.write(u'\n')
expanded_file.close()
chmod_plus_x(expanded_name)
@@ -99,7 +94,8 @@ def move_file_to_backup(file_name, backup_name):
# Namelist formatting
-quote = repr
+def quote(value):
+ return repr(value).lstrip('u')
def format_atom(value):
'''Format atomic value for use in namelists'''
@@ -133,7 +129,7 @@ def format_value(value, indent):
sep = ', '
if line:
lines.append(line)
- return (',\n' + ' '*indent).join(lines)
+ return (u',\n' + ' '*indent).join(lines)
return format_atom(value)
def keyword_warning(key):
@@ -145,7 +141,7 @@ def get_remove_list(section, key):
# Deprecation warning for non .keys
if key[0] != '.':
keyword_warning(key)
- if isinstance(section[key], basestring):
+ if not isinstance(section[key], (list, tuple)):
remove_list = [key, section[key]]
else:
remove_list = [key] + section[key]
@@ -159,8 +155,8 @@ def format_namelist_comment(line):
comment = ''
if match.group(4):
comment = ' ! '+match.group(4)
- return '! '+key+' = '+format_value(value, 0)+comment+'\n'
- return re.sub(r'^#', '!', line)+'\n'
+ return '! '+key+' = '+format_value(value, 0)+comment+u'\n'
+ return re.sub(r'^#', '!', line)+u'\n'
def format_namelist(section, group=None, default_value=''):
'''Format config section as a namelist.
@@ -182,17 +178,16 @@ def format_namelist(section, group=None, default_value=''):
# Support old keyword for backward compatibility
remove_list = get_remove_list(section, '.remove')
remove_list += get_remove_list(section, 'remove')
- black_list = map(lambda x: x.replace(r'\*', '.*').replace(r'\?', '.')+'$',
- map(lambda x: re.escape(x.lower()), remove_list))
+ black_list = [x.replace(r'\*', '.*').replace(r'\?', '.')+'$' for x in [re.escape(x.lower()) for x in remove_list]]
# Format namelist groups that were not removed
- lines = StringIO.StringIO()
- iterator = {group: section[group]}.iteritems() if group else section.iteritems()
+ lines = io.StringIO()
+ iterator = iter({group: section[group]}.items()) if group else iter(section.items())
for group, contents in iterator:
if isinstance(contents, dict):
hidden = is_set(contents.get(hide_key))
group_def_val = contents.get(default_key, default_value)
group_id = group.lower()
- if not hidden and not any(map(lambda x: re.match(x, group_id), black_list)):
+ if not hidden and not any([re.match(x, group_id) for x in black_list]):
# Create list of removed keys
remove_keys = get_remove_list(contents, '.remove')
# Start namelist group
@@ -200,16 +195,17 @@ def format_namelist(section, group=None, default_value=''):
lines.write(format_namelist_comment(line))
group_names = group_id.split(' ', 1)
if len(group_names) == 1:
- lines.write('&'+group_names[0]+'\n')
+ lines.write(u'&'+group_names[0]+u'\n')
else:
- lines.write('&'+group_names[0]+" ! '"+group_names[1]+"'\n")
- for key, value in contents.iteritems():
+ lines.write('&'+group_names[0]+" ! '"+group_names[1]
+ +u"'\n")
+ for key, value in contents.items():
if (key[0] != '.' and key not in remove_keys and
value != group_def_val):
key = key.lower()
indent = base_indent + len(key) + 3
for line in contents.comments.get(key, []):
- lines.write(' '*base_indent)
+ lines.write(u' '*base_indent)
lines.write(format_namelist_comment(line))
line = contents.inline_comments[key]
if not line:
@@ -217,12 +213,12 @@ def format_namelist(section, group=None, default_value=''):
line = re.sub(r'^#', ' !', line)
lines.write(' '*base_indent+key+' = '+
format_value(value, indent)+
- line+'\n')
+ line+u'\n')
if end_key in contents:
for line in contents.comments[end_key]:
- lines.write(' '*base_indent)
+ lines.write(u' '*base_indent)
lines.write(format_namelist_comment(line))
- lines.write('/\n')
+ lines.write(u'/\n')
return lines.getvalue()
# Global formatting
@@ -238,12 +234,12 @@ def format_vars(section, key, log, fmt):
value = section[key]
newkey = transform(key)
section.rename(key, newkey)
- if isinstance(value, basestring):
+ if not isinstance(value, (list, tuple, dict)):
# Format string variables
section[newkey] = transform(value)
- elif '__iter__' in dir(value) and not isinstance(value, dict):
+ elif isinstance(value, (list, tuple)):
# Format all list elements
- section[newkey] = map(transform, value)
+ section[newkey] = list(map(transform, value))
#
# Main routine
@@ -277,7 +273,7 @@ if not os.path.exists(experiment_config_name):
feedback.die("config file '{0}' does not exist".format(experiment_config_name))
# Overrides
-invalid_args = filter(lambda x: not x.find('=')+1, args.assigns)
+invalid_args = [x for x in args.assigns if not x.find('=')+1]
if invalid_args:
feedback.die("invalid parameters ('"+"', '".join(invalid_args)+"')\n" +
@@ -286,15 +282,15 @@ if invalid_args:
# Setup templating environment
template_env = Environment(
- loader = ChoiceLoader(map(FileSystemLoader, config_roots)),
+ loader = ChoiceLoader(list(map(FileSystemLoader, config_roots))),
variable_start_string = '%{',
variable_end_string = '}',
line_statement_prefix = '#%',
line_comment_prefix = '#%#',
block_start_string = '{%__mkexp__',
comment_start_string = '{#__mkexp__',
- extensions=['jinja2.ext.do']
-)
+ extensions = ['jinja2.ext.do']
+)
# Additional global functions
@@ -337,12 +333,18 @@ template_env.filters['match'] = match
template_env.filters['split'] = lambda x, s=None, m=-1: x.split(s, m)
# - Add list operation filter
-template_env.filters['filter'] = lambda x, f=None: filter(f, x)
+template_env.filters['filter'] = lambda x, f=None: list(filter(f, x))
# - Replace 'list' handling simple values and strings as singleton lists
list_original = template_env.filters['list']
-def list_singleton(x, keep_empty=False, *args, **kwargs):
- if '__iter__' in dir(x):
+@jinja2.evalcontextfilter
+def list_singleton(eval_ctx, x, keep_empty=False, *args, **kwargs):
+ # Workaround for 2.8 bug when applied to literals
+ if isinstance(x, jinja2.nodes.EvalContext):
+ (eval_ctx, x) = (x, eval_ctx)
+ if isinstance(x, (list, tuple)):
+ if getattr(list_original, 'evalcontextfilter', False):
+ return list_original(eval_ctx, x, *args, **kwargs)
return list_original(x, *args, **kwargs)
if not keep_empty and x == '':
return []
@@ -356,7 +358,7 @@ def join_singleton(eval_ctx, x, *args, **kwargs):
# Workaround for 2.8 bug when applied to literals
if isinstance(x, jinja2.nodes.EvalContext):
(eval_ctx, x) = (x, eval_ctx)
- if '__iter__' in dir(x):
+ if isinstance(x, (list, tuple)):
return join_original(eval_ctx, x, *args, **kwargs)
return x
template_env.filters['join'] = join_singleton
@@ -393,7 +395,7 @@ def cut_dir_variable(directory):
# Create directory for scripts if it doesn't exist
script_dir = config['SCRIPT_DIR']
-print "Script directory: '"+script_dir+"'"
+print("Script directory: '"+script_dir+"'")
time_stamp = strftime("%Y%m%d%H%M%S")
backup_dir = os.path.join(script_dir, 'backup', time_stamp)
if not os.path.isdir(script_dir):
@@ -406,8 +408,8 @@ else:
# Create directory for output data if it doesn't exist
data_dir = config['DATA_DIR']
data_cut = cut_dir_variable(data_dir)
-print "Data directory: '"+data_dir+"'"+(
- " (not created)" if not args.make_dirs else "")
+print("Data directory: '"+data_dir+"'"+(
+ " (not created)" if not args.make_dirs else ""))
if args.make_dirs:
if data_dir != data_cut:
feedback.warning("only considering non-variable part of directory")
@@ -420,8 +422,8 @@ if args.make_dirs:
work_dir = config['WORK_DIR']
work_cut = cut_dir_variable(work_dir)
if work_dir != data_dir:
- print "Work directory: '"+work_dir+"'"+(
- " (not created)" if not args.make_dirs else "")
+ print("Work directory: '"+work_dir+"'"+(
+ " (not created)" if not args.make_dirs else ""))
if args.make_dirs:
if work_dir != work_cut:
feedback.warning("only considering non-variable part of directory")
@@ -435,7 +437,7 @@ if work_dir != data_dir:
dump_name = config.experiment_id+'.dump'
move_file_to_backup(os.path.join(script_dir, dump_name),
os.path.join(backup_dir, dump_name))
-dump_file = open_utf8(os.path.join(script_dir, dump_name), 'w')
+dump_file = io.open(os.path.join(script_dir, dump_name), 'wb')
default_indent_type = config.indent_type
config.indent_type = ' '
config.write(dump_file)
@@ -450,7 +452,7 @@ dump_file.close()
job_dict = {}
remove_list = get_remove_list(config['jobs'], '.remove')
remove_list += get_remove_list(config['jobs'], 'remove')
-for key, value in config['jobs'].iteritems():
+for key, value in config['jobs'].items():
if not isinstance(value, dict):
job_dict[key] = value
del config['jobs'][key]
@@ -483,7 +485,7 @@ def extend(subjob, jobs_config, extended_jobs):
# Add actual subjob config
pre_config.merge(subconfig)
- # Replace subjob config by extended config_lines
+ # Replace subjob config by extended config
jobs_config[subjob] = {}
jobs_config[subjob].merge(pre_config)
del pre_config
@@ -503,20 +505,19 @@ for subjob in jobs_config:
subconfig = jobs_config[subjob]
subconfig['id'] = subjob
if not 'tasks' in subconfig:
- subconfig['tasks'] = int(subconfig.get('nodes', 1)) * \
- int(subconfig.get('tasks_per_node', 1))
-
-# Save configuration to buffer, to be merged with each job config
-config_lines = StringIO.StringIO()
-config.write(config_lines)
+ subconfig['tasks'] = str(int(subconfig.get('nodes', 1)) *
+ int(subconfig.get('tasks_per_node', 1)))
# Paste them into each job
-for subjob, subconfig in jobs_config.iteritems():
+for subjob, subconfig in jobs_config.items():
if not subjob in remove_list:
# Copy current config settings to job
+ config_lines = io.BytesIO()
+ config.write(config_lines)
config_lines.seek(0)
- job_config = ConfigObj(config_lines, interpolation=False)
+ job_config = ConfigObj(io.TextIOWrapper(config_lines),
+ interpolation=False)
# Check namelist override
if 'namelists' in subconfig:
@@ -529,7 +530,7 @@ for subjob, subconfig in jobs_config.iteritems():
del subconfig['files']
# Paste pre config into job config
- job_config['JOB'] = subconfig
+ job_config[u'JOB'] = subconfig
del subconfig
# Prepare namelists for inclusion in scripts
@@ -539,7 +540,7 @@ for subjob, subconfig in jobs_config.iteritems():
job_config['jobs'] = copy.deepcopy(jobs_config)
job_config.walk(format_vars, log=var_list, fmt=var_format)
job_config['VARIABLES_'] = var_list
- for namelist, groups in job_config['namelists'].iteritems():
+ for namelist, groups in job_config['namelists'].items():
if isinstance(groups, dict):
# Skip hidden namelists
if is_set(groups.get('.hide')):
@@ -587,27 +588,27 @@ for subjob, subconfig in jobs_config.iteritems():
move_file_to_backup(os.path.join(script_dir, 'README'),
os.path.join(backup_dir, 'README'))
-readme_file = open_utf8(os.path.join(script_dir, 'README'), 'w')
-readme_file.write(config['EXP_DESCRIPTION'] + '\n')
+readme_file = io.open(os.path.join(script_dir, 'README'), 'w')
+readme_file.write(config['EXP_DESCRIPTION'] + u'\n')
readme_file.close()
# Create update script from experiment description
move_file_to_backup(os.path.join(script_dir, 'update'),
os.path.join(backup_dir, 'update'))
-update_file = open_utf8(os.path.join(script_dir, 'update'), 'w')
-update_file.write('#! /bin/sh\n')
-update_file.write('#\n')
-update_file.write('# Regenerate all files with identical configuration\n')
-update_file.write('#\n')
-update_file.write('# ' + extra_dict['mkexp_input'].replace('$$', '$') + '\n')
-update_file.write('#\n')
-update_file.write('cd ' + quote(os.environ.get('PWD', os.getcwd())) + '\n')
-update_file.write('PATH=' + quote(os.environ.get('PATH', '')) + '\n')
-update_file.write('PYTHONPATH=' + quote(os.environ.get('PYTHONPATH', '')) + '\n')
-update_file.write('MKEXP_PATH=' + quote(os.environ.get('MKEXP_PATH', '')) + '\n')
-update_file.write('export PATH PYTHONPATH MKEXP_PATH\n')
-update_file.write('echo ' + ' '.join(map(quote, sys.argv)) + ' "$@" >&2\n')
-update_file.write('exec ' + ' '.join(map(quote, sys.argv)) + ' "$@"\n')
+update_file = io.open(os.path.join(script_dir, 'update'), 'w')
+update_file.write(u'#! /bin/sh\n')
+update_file.write(u'#\n')
+update_file.write(u'# Regenerate all files with identical configuration\n')
+update_file.write(u'#\n')
+update_file.write(u'# ' + extra_dict['mkexp_input'].replace('$$', '$') + '\n')
+update_file.write(u'#\n')
+update_file.write(u'cd ' + quote(os.environ.get('PWD', os.getcwd())) + '\n')
+update_file.write(u'PATH=' + quote(os.environ.get('PATH', '')) + '\n')
+update_file.write(u'PYTHONPATH=' + quote(os.environ.get('PYTHONPATH', '')) + '\n')
+update_file.write(u'MKEXP_PATH=' + quote(os.environ.get('MKEXP_PATH', '')) + '\n')
+update_file.write(u'export PATH PYTHONPATH MKEXP_PATH\n')
+update_file.write(u'echo ' + ' '.join(map(quote, sys.argv)) + ' "$@" >&2\n')
+update_file.write(u'exec ' + ' '.join(map(quote, sys.argv)) + ' "$@"\n')
update_file.close()
chmod_plus_x(os.path.join(script_dir, 'update'))
diff --git a/rmexp b/rmexp
index 79ac102ba56c6f588c571ede5ba11eefa915d75b..7d11bccfa9507ed69c302d61c29716f5493be910 100755
--- a/rmexp
+++ b/rmexp
@@ -9,6 +9,9 @@ PROGRAM=`basename $0`
BINDIR=`dirname $0`
PATH="$BINDIR:$PATH"
+PYTHONIOENCODING=$(locale -c charmap | tail -1)
+export PYTHONIOENCODING
+
die () {
echo "$@" >&2
exit 1
diff --git a/selconfig b/selconfig
index ea9822121a9fbc681646389f7decc3dd34914fd2..3ba0c35490acec4e85a3b9272be9e36a239acf3c 100755
--- a/selconfig
+++ b/selconfig
@@ -5,12 +5,14 @@ Select the given section of a config file
$Id$
'''
+from __future__ import print_function
+
import argparse
+import io
import re
import sys
-import StringIO
-from configobj import ConfigObj
+from expconfig import ConfigObj
from expargparse import assigns_to_dicts, get_key_chain
from feedback import die
@@ -51,28 +53,31 @@ except IOError as error:
selected_data = ConfigObj(interpolation=False, write_empty_values=True,
indent_type=' ')
-config = config_data
-selected = selected_data
-for section in reversed(get_key_chain(args.section)):
- if section in config.comments:
- selected.comments[section] = config.comments[section]
- if section in config.inline_comments:
- selected.inline_comments[section] = config.inline_comments[section]
- config = config[section]
- selected[section] = {}
- selected = selected[section]
-# Replace the empty leaf section by the original section to be selected
-selected.parent[section] = config
+if args.section:
+ config = config_data
+ selected = selected_data
+ for section in reversed(get_key_chain(args.section)):
+ if section in config.comments:
+ selected.comments[section] = config.comments[section]
+ if section in config.inline_comments:
+ selected.inline_comments[section] = config.inline_comments[section]
+ config = config[section]
+ selected[section] = {}
+ selected = selected[section]
+ # Replace the empty leaf section by the original section to be selected
+ selected.parent[section] = config
+else:
+ selected_data.merge(config_data)
# Ready to roll out
-lines = StringIO.StringIO()
+lines = io.BytesIO()
selected_data.write(lines)
lines.seek(0)
-for line in lines:
+for line in io.TextIOWrapper(lines):
if args.inline_comments: line = re.sub(r' = (.*?) #', r' = \1 #', line)
if args.trailing_space:
print(line.rstrip())
else:
- sys.stdout.write(line)
+ print(line, end='')
diff --git a/setconfig b/setconfig
index d174d04496ce66389b8d1034e399e745b07d8678..6060a0552a76d66b55cca4de92751562513de5c0 100755
--- a/setconfig
+++ b/setconfig
@@ -1,16 +1,18 @@
#! /usr/bin/env python
-#
-# Change configuration using command line
-#
-# $Id$
-#
+'''\
+Change configuration using command line
+
+$Id$
+'''
+
+from __future__ import print_function
import argparse
+import io
import re
import sys
-import StringIO
-from configobj import ConfigObj
+from expconfig import ConfigObj
from expargparse import assigns_to_dicts, get_key_chain
from feedback import die
@@ -22,8 +24,7 @@ import package_info
# Check command line
-command_line = argparse.ArgumentParser(description=
- 'Change configuration using command line.')
+command_line = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0])
command_line.add_argument('config', nargs='?', default='-',
help='original configuration file name [%(default)s]')
command_line.add_argument('assigns', metavar='key=value', nargs='*',
@@ -45,6 +46,7 @@ command_line.add_argument('--trailing-space' , '-t', action='store_true',
args = command_line.parse_args()
# File handling
+
try:
config_file = args.config
if config_file == '-':
@@ -94,12 +96,13 @@ for line in back_plate:
# Ready to roll out
-lines = StringIO.StringIO()
+lines = io.BytesIO()
config_data.write(lines)
+
lines.seek(0)
-for line in lines:
+for line in io.TextIOWrapper(lines):
if args.inline_comments: line = re.sub(r' = (.*?) #', r' = \1 #', line)
if args.trailing_space:
print(line.rstrip())
else:
- sys.stdout.write(line)
+ print(line, end='')
diff --git a/test.py b/test.py
index c06e83adfe046eed2dae198e0c471f280781ec40..4988533034500b9ef300a1750a9615045e2a4187 100644
--- a/test.py
+++ b/test.py
@@ -1,35 +1,53 @@
+# -*- coding: utf-8 -*-
+
+import io
+import locale
import os
import re
+import subprocess
import sys
import unittest
from os.path import join
+_preferred_encoding = locale.getpreferredencoding()
def align(string):
return re.sub(r'\n\s*', '\n', string.lstrip())
-def script(string):
- return align("""
+def script(string, set_encoding=False):
+ text = u"""
set -e
unset CDPATH
cd test
- PATH=..:$PATH
+ PATH=..:.:$PATH
MKEXP_PATH=
- """) + align(string)
+ """
+ if set_encoding and sys.getdefaultencoding() == 'ascii':
+ text += u"""
+ PYTHONIOENCODING={0}
+ export PYTHONIOENCODING
+ """.format(_preferred_encoding)
+ text += string
+ return align(text)
def output(command):
- (ignore, stream) = os.popen4(command)
- return stream.read()
+ p = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ (result, ignore) = p.communicate()
+ try: result = str(result, _preferred_encoding)
+ except TypeError: pass
+ ### print("DEBUG:\n" + result) ###
+ return result
def readfile(filename):
- stream = open(filename)
- return stream.read()
+ with io.open(filename) as stream:
+ return stream.read()
def writefile(filename, string):
- stream = open(filename, 'w')
- stream.write(string)
+ with io.open(filename, 'w') as stream:
+ stream.write(string)
def writeconfig(exp_id, string):
writefile(join("test", exp_id+".config"), string)
@@ -41,12 +59,12 @@ def writetemplate(exp_id, job_id, string):
class MkexpTestCase(unittest.TestCase):
- script_clean = script("""
+ script_clean = script(u"""
rm -rf experiments
rm -f test_* SETUP.config
""")
- script_run = script("""
+ script_run = script(u"""
mkexp test0001.config
""")
@@ -65,11 +83,11 @@ class MkexpSimpleTestCase(MkexpTestCase):
MkexpTestCase.setUp(self)
def run_test(self, template, expected, additional=''):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
EXP_TYPE =
- """+additional+"""
+ """+additional+u"""
[jobs]
- [["""+self.job_id+"""]]
+ [["""+self.job_id+u"""]]
""")
writetemplate(self.exp_id, self.job_id, template)
expected = align(expected)
@@ -80,11 +98,11 @@ class MkexpSimpleTestCase(MkexpTestCase):
self.assertMultiLineEqual(expected, result)
def run_no_template(self, result_path, expected, additional=''):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
EXP_TYPE =
- """+additional+"""
+ """+additional+u"""
[jobs]
- [["""+self.job_id+"""]]
+ [["""+self.job_id+u"""]]
""")
expected = align(expected)
ignore = output(script("mkexp "+self.exp_id+".config"))
@@ -101,7 +119,7 @@ class RunningTestCase(MkexpTestCase):
self.assertIn('error: too few arguments', result)
def test_clean_run(self):
- expected = align("""
+ expected = align(u"""
Script directory: 'experiments/test0001'
Data directory: 'experiments/test0001'
""")
@@ -118,7 +136,7 @@ class RunningTestCase(MkexpTestCase):
class CommandLineTestCase(MkexpTestCase):
def test_pass_section_variables(self):
- script_section = script("""
+ script_section = script(u"""
mkexp test0001.config \
namelists.namelist..echam.runctl.dt_start=2345,01,23,12,34,56 \
namelists.namelist..echam.runctl.some_file=abcdefgh.ijk
@@ -136,7 +154,7 @@ class CommandLineTestCase(MkexpTestCase):
# Should exist, otherwise exception is thrown
def test_options(self):
- script_option = script("""
+ script_option = script(u"""
mkexp test0001.config EXP_OPTIONS=option1
""")
expected = "default_output = .false."
@@ -145,12 +163,12 @@ class CommandLineTestCase(MkexpTestCase):
self.assertIn(expected, result)
def test_getexp_vv(self):
- script_getexp = script("""
+ script_getexp = script(u"""
mkexp test0001.config
mv experiments/test0001 experiments/test0001.orig
getexp -vv test0001.config MODEL_DIR=. > test_getexp.dump
mkexp --getexp test_getexp.dump
- """)
+ """, set_encoding=True)
ignore = output(script_getexp)
expected = readfile('test/experiments/test0001.orig/test0001.run')
result = readfile('test/experiments/test0001/test0001.run')
@@ -158,7 +176,7 @@ class CommandLineTestCase(MkexpTestCase):
def test_getexp_k(self):
result = output(script('getexp -k VAR1 test0001.config'))
- expected = align("""
+ expected = align(u"""
Note: data for experiment 'test0001' does not exist
value1
""")
@@ -166,7 +184,7 @@ class CommandLineTestCase(MkexpTestCase):
def test_getexp_k_k(self):
result = output(script('getexp -k VAR1 -k VAR2 test0001.config'))
- expected = align("""
+ expected = align(u"""
Note: data for experiment 'test0001' does not exist
value1
value1
@@ -175,25 +193,55 @@ class CommandLineTestCase(MkexpTestCase):
def test_getexp_s(self):
result = output(script('getexp -s -k VAR1 test0001.config'))
- expected = align("""
+ expected = align(u"""
Note: data for experiment 'test0001' does not exist
VAR1='value1'
""")
self.assertMultiLineEqual(expected, result)
def test_getconfig(self):
- ignore = output(script("""
+ ignore = output(script(u"""
mkexp test0001.config VAR4=value4 jobs.run.time_limit=12:34:56
"""))
- result = output(script('getconfig experiments/test0001/update'))
+ result = output(script('getconfig experiments/test0001/update',
+ set_encoding=True))
self.assertIn('VAR4 = value4', result)
self.assertIn('time_limit = 12:34:56', result)
+ def test_setconfig(self):
+ result = output(script(u"""
+ setconfig test0001.config VAR4=value4 jobs.run.time_limit=12:34:56
+ """, set_encoding=True))
+ self.assertIn('VAR4 = value4', result)
+ self.assertIn('time_limit = 12:34:56', result)
+
+ def test_selconfig(self):
+ result = output(script(u"""
+ selconfig VAR1 test0001.config
+ """))
+ self.assertIn('VAR1 = value1', result)
+ self.assertNotIn('VAR2', result)
+
+ def test_rmexp(self):
+ script_getexp = script(u"""
+ mkexp test0001.config
+ (echo n; echo y) | rmexp test0001.config
+ """)
+ result = output(script_getexp)
+ self.assertIn("Script directory: 'experiments/test0001'", result)
+ self.assertIn("Data directory: 'experiments/test0001'", result)
+ self.assertIn("Info for test0001's working directory:", result)
+ self.assertIn("rmexp: remove test0001's working directory and its contents [no]? Info for test0001's script directory:", result)
+ self.assertNotIn("rmexp: remove test0001's working directory and its contents [no]? rmexp: removed 'experiments/test0001'", result)
+ self.assertIn("rmexp: remove test0001's script directory and its contents [no]? rmexp: removed 'experiments/test0001'", result)
+ self.assertNotIn("rmexp: remove test0001's script directory and its contents [no]? Info for test0001's data directory:", result)
+ self.assertNotIn("rmexp: remove test0001's data directory and its contents [no]?", result)
+
class ContentTestCase(MkexpSimpleTestCase):
def test_job_override(self):
exp_id = "test_job_override"
- writeconfig(exp_id, """
+ writeconfig(exp_id, u"""
EXP_TYPE =
[jobs]
key1 = global
@@ -201,7 +249,7 @@ class ContentTestCase(MkexpSimpleTestCase):
[[job1]]
key1 = local
""")
- writetemplate(exp_id, "job1", """
+ writetemplate(exp_id, "job1", u"""
key1 = %{JOB.key1}
key2 = %{JOB.key2}
""")
@@ -213,7 +261,7 @@ class ContentTestCase(MkexpSimpleTestCase):
def test_var_list_in_context(self):
exp_id = "test_var_list_in_context"
job_id = "job"
- writeconfig(exp_id, """
+ writeconfig(exp_id, u"""
EXP_TYPE =
VAR1 = value1
GLOBAL1 = $${VAR1} # Initialized
@@ -221,14 +269,14 @@ class ContentTestCase(MkexpSimpleTestCase):
GLOBAL3 = $${VAR1} # Used twice, may only be listed once
GLOBAL${FOUR} = 4 # (Uninitialized) Variable in key
[jobs]
- [["""+job_id+"""]]
+ [["""+job_id+u"""]]
""")
- writetemplate(exp_id, job_id, """
- #% for var in VARIABLES_:
+ writetemplate(exp_id, job_id, u"""
+ #% for var in VARIABLES_|sort:
%{var}=%{context(var)}
#% endfor
""")
- expected = align("""
+ expected = align(u"""
FOUR=
VAR1=value1
VAR2=
@@ -241,20 +289,20 @@ class ContentTestCase(MkexpSimpleTestCase):
def test_split_date(self):
exp_id = 'test_split_date'
job_id = 'job'
- writeconfig(exp_id, """
+ writeconfig(exp_id, u"""
EXP_TYPE =
DATE_ISO = 1234-05-06
DATE_RAW = 12340506
DATE_LIST_ISO = split_date($DATE_ISO)
DATE_LIST_RAW = split_date($DATE_RAW)
[jobs]
- [["""+job_id+"""]]
+ [["""+job_id+u"""]]
""")
- writetemplate(exp_id, job_id, """
+ writetemplate(exp_id, job_id, u"""
%{DATE_LIST_ISO|join(',')}
%{DATE_LIST_RAW|join(',')}
""")
- expected = align("""
+ expected = align(u"""
1234,5,6,0,0,0
1234,05,06,0,0,0
""")
@@ -266,7 +314,7 @@ class ContentTestCase(MkexpSimpleTestCase):
def test_add_years(self):
exp_id = 'test_add_years'
job_id = 'job'
- writeconfig(exp_id, """
+ writeconfig(exp_id, u"""
EXP_TYPE =
DATE = 1234-05-06
NEXT_DATE = 'add_years($DATE, 1)'
@@ -274,15 +322,15 @@ class ContentTestCase(MkexpSimpleTestCase):
NEGATIVE_DATE = 'add_years($DATE, -2000)'
LONGYEAR_DATE = 'add_years($DATE, 10000)'
[jobs]
- [["""+job_id+"""]]
+ [["""+job_id+u"""]]
""")
- writetemplate(exp_id, job_id, """
+ writetemplate(exp_id, job_id, u"""
%{NEXT_DATE}
%{PREVIOUS_DATE}
%{NEGATIVE_DATE}
%{LONGYEAR_DATE}
""")
- expected = align("""
+ expected = align(u"""
1235-05-06
1233-05-06
-0766-05-06
@@ -296,7 +344,7 @@ class ContentTestCase(MkexpSimpleTestCase):
def test_add_days(self):
exp_id = 'test_add_days'
job_id = 'job'
- writeconfig(exp_id, """
+ writeconfig(exp_id, u"""
EXP_TYPE =
DATE = 1234-05-06
NEXT_DATE = 'add_days($DATE, 1)'
@@ -308,9 +356,9 @@ class ContentTestCase(MkexpSimpleTestCase):
EARLY_DATE = 0000-01-01
EARLIER_DATE = 'add_days($EARLY_DATE, -1)'
[jobs]
- [["""+job_id+"""]]
+ [["""+job_id+u"""]]
""")
- writetemplate(exp_id, job_id, """
+ writetemplate(exp_id, job_id, u"""
%{NEXT_DATE}
%{PREVIOUS_DATE}
%{NEGATIVE_DATE}
@@ -318,7 +366,7 @@ class ContentTestCase(MkexpSimpleTestCase):
%{LATER_DATE}
%{EARLIER_DATE}
""")
- expected = align("""
+ expected = align(u"""
1234-05-07
1234-05-05
1228-11-13
@@ -332,37 +380,37 @@ class ContentTestCase(MkexpSimpleTestCase):
self.assertMultiLineEqual(expected, result)
def test_eval(self):
- self.run_test("""
+ self.run_test(u"""
%{VALUE}
- """, """
+ """, u"""
42
- """, """
+ """, u"""
VALUE = eval(5*8+2)
""")
def test_eval_time(self):
- self.run_test("""
+ self.run_test(u"""
%{VALUE}
- """, """
+ """, u"""
1970-01-01
- """, """
+ """, u"""
VALUE = "eval(time.strftime('%Y-%m-%d', time.gmtime(0)))"
""")
def test_eval_is_set(self):
- self.run_test("""
+ self.run_test(u"""
%{TRUE}
%{FALSE}
- """, """
+ """, u"""
True
False
- """, """
+ """, u"""
TRUE = eval(is_set('.true.'))
FALSE = eval(is_set('F'))
""")
def test_initial_comment_boilerplate(self):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
######
# #
# 42
@@ -372,7 +420,7 @@ class ContentTestCase(MkexpSimpleTestCase):
[jobs]
[["""+self.job_id+"""]]
""")
- writetemplate(self.exp_id, self.job_id, """
+ writetemplate(self.exp_id, self.job_id, u"""
%{EXP_DESCRIPTION}
""")
expected = align("""
@@ -387,9 +435,9 @@ class ContentTestCase(MkexpSimpleTestCase):
class NamelistTestCase(MkexpSimpleTestCase):
def test_namelist_comments(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- ""","""
+ """,u"""
! Comment group 1
! var_1c = 'test'
&group_1
@@ -402,7 +450,7 @@ class NamelistTestCase(MkexpSimpleTestCase):
! Comment for var 2b
var_2b = 21 ! Inline comment for var 2b
/
- ""","""
+ """,u"""
[namelists]
[[namelist]]
# Comment group 1
@@ -419,9 +467,9 @@ class NamelistTestCase(MkexpSimpleTestCase):
""")
def test_var_in_namelist(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- ""","""
+ """,u"""
&group
var_1 = $value_1
var_2 = ${value_2}
@@ -429,7 +477,7 @@ class NamelistTestCase(MkexpSimpleTestCase):
var_4 = 'a$value_4'
var_5 = '${value_5}b'
/
- ""","""
+ """,u"""
[namelists]
[[namelist]]
[[[group]]]
@@ -441,9 +489,9 @@ class NamelistTestCase(MkexpSimpleTestCase):
""")
def test_namelist_multi_groups(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- """, """
+ """, u"""
&group ! '1'
/
&group ! ' 1'
@@ -452,7 +500,7 @@ class NamelistTestCase(MkexpSimpleTestCase):
/
&group ! 'i i i'
/
- """, """
+ """, u"""
[namelists]
[[namelist]]
[[[group 1]]]
@@ -462,14 +510,14 @@ class NamelistTestCase(MkexpSimpleTestCase):
""")
def test_namelist_case_twist(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- """, """
+ """, u"""
&group
value = 41
value = 42
/
- """, """
+ """, u"""
[namelists]
[[namelist]]
[[[group]]]
@@ -478,11 +526,11 @@ class NamelistTestCase(MkexpSimpleTestCase):
""")
def test_namelist_format(self):
- self.run_test("""
+ self.run_test(u"""
%{format_namelist(namelists.namelist)}
%{format_namelist(namelists.namelist, 'group2')}
%{format_namelist(namelists.namelist, 'no such group')}
- """, """
+ """, u"""
&group1
value = 41
/
@@ -492,7 +540,7 @@ class NamelistTestCase(MkexpSimpleTestCase):
&group2
value = 42
/
- """, """
+ """, u"""
[namelists]
[[namelist]]
.remove = group3
@@ -505,11 +553,11 @@ class NamelistTestCase(MkexpSimpleTestCase):
""")
def test_namelist_with_template(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- """, """
+ """, u"""
42
- """, """
+ """, u"""
[namelists]
[[namelist]]
.use_template = true
@@ -520,16 +568,16 @@ class NamelistTestCase(MkexpSimpleTestCase):
class NamelistHiddenTestCase(MkexpSimpleTestCase):
def test_namelist_hide(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- """, """
+ """, u"""
&group1
value = 41
/
&group3
value = 43
/
- """, """
+ """, u"""
[namelists]
[[namelist]]
[[[group1]]]
@@ -543,10 +591,10 @@ class NamelistHiddenTestCase(MkexpSimpleTestCase):
""")
def test_hidden_namelist_file(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- """, """
- """, """
+ """, u"""
+ """, u"""
[namelists]
[[namelist]]
.hide = true
@@ -555,10 +603,10 @@ class NamelistHiddenTestCase(MkexpSimpleTestCase):
""")
def test_hidden_namelist_with_template(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- """, """
- """, """
+ """, u"""
+ """, u"""
[namelists]
[[namelist]]
.hide = true
@@ -570,12 +618,12 @@ class NamelistHiddenTestCase(MkexpSimpleTestCase):
class NamelistDefaultValueTestCase(MkexpSimpleTestCase):
def test_standard_default_value(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- """, """
+ """, u"""
&group
/
- """, """
+ """, u"""
[namelists]
[[namelist]]
[[[group]]]
@@ -583,13 +631,13 @@ class NamelistDefaultValueTestCase(MkexpSimpleTestCase):
""")
def test_custom_default_value(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST}
- """, """
+ """, u"""
&group
value1 = ''
/
- """, """
+ """, u"""
[namelists]
[[namelist]]
.default = <DEFAULT>
@@ -599,16 +647,16 @@ class NamelistDefaultValueTestCase(MkexpSimpleTestCase):
""")
def test_namelist_default_value(self):
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST1}
%{NAMELIST2}
- """, """
+ """, u"""
&group
/
&group
value = '<DEFAULT>'
/
- """, """
+ """, u"""
[namelists]
[[namelist1]]
.default = <DEFAULT>
@@ -620,12 +668,12 @@ class NamelistDefaultValueTestCase(MkexpSimpleTestCase):
""")
def test_global_default_value(self):
- self.run_test("""
+ self.run_test(u"""
%{format_namelist(namelists.namelist, default_value='<DEFAULT>')}
- """, """
+ """, u"""
&group
/
- """, """
+ """, u"""
[namelists]
[[namelist]]
[[[group]]]
@@ -635,17 +683,17 @@ class NamelistDefaultValueTestCase(MkexpSimpleTestCase):
class JinjaTemplateTestCase(MkexpSimpleTestCase):
def test_ignore_blocks(self):
- self.run_test("""
+ self.run_test(u"""
{% set answer = 42 %}
- """, """
+ """, u"""
{% set answer = 42 %}
""")
def test_ignore_comments(self):
- self.run_test("""
+ self.run_test(u"""
{# no comment #}
${#ARRAY}
- """, """
+ """, u"""
{# no comment #}
${#ARRAY}
""")
@@ -653,111 +701,111 @@ class JinjaTemplateTestCase(MkexpSimpleTestCase):
class DefaultEnvironmentTestCase(MkexpSimpleTestCase):
def test_basic(self):
- self.run_test("""
+ self.run_test(u"""
%{ENVIRONMENT}
- """, """
+ """, u"""
DEFAULT
""")
def test_explicit(self):
- self.run_test("""
+ self.run_test(u"""
%{ENVIRONMENT}
- """, """
+ """, u"""
green
- """, """
+ """, u"""
ENVIRONMENT = green
""")
def test_setup(self):
- writeconfig('SETUP', """
+ writeconfig('SETUP', u"""
ENVIRONMENT =
""")
- self.run_test("""
+ self.run_test(u"""
%{ENVIRONMENT}
- """, """
+ """, u"""
DEFAULT
""")
class SetupConfigTestCase(MkexpSimpleTestCase):
def test_system_options(self):
- writeconfig('SETUP', """
+ writeconfig('SETUP', u"""
SETUP_OPTIONS = option1
""")
- self.run_test("""
+ self.run_test(u"""
%{NAMELIST_ECHAM}
- """, """
+ """, u"""
&runctl
default_output = .false.
/
- """, """
+ """, u"""
EXP_OPTIONS =
""")
class MatchTestCase(MkexpSimpleTestCase):
def test_basic(self):
- self.run_test("""
+ self.run_test(u"""
%{'Douglas Adams'|match('Adam')}
- """, """
+ """, u"""
Douglas Adams
""")
def test_no_match(self):
- self.run_test("""
+ self.run_test(u"""
%{'Douglas Adams'|match('Eva')}
- """, """
+ """, u"""
""")
def test_with_default(self):
- self.run_test("""
+ self.run_test(u"""
%{'Douglas Adams'|match('Abel', 'Kain')}
- """, """
+ """, u"""
Kain
""")
def test_with_group(self):
- self.run_test("""
+ self.run_test(u"""
%{'Douglas Adams'|match('l(.*)m')}
- """, """
+ """, u"""
as Ada
""")
class SplitTestCase(MkexpSimpleTestCase):
def test_basic(self):
- self.run_test("""
+ self.run_test(u"""
%{'Douglas Noel Adams'|split(' ')}
%{'Douglas Noel Adams'|split(' ')}
- ""","""
+ """,u"""
['Douglas', 'Noel', '', 'Adams']
['Douglas Noel', 'Adams']
""")
def test_max_split(self):
- self.run_test("""
+ self.run_test(u"""
%{'Douglas Noel Adams'|split(' ', 2)}
%{'Douglas Noel Adams'|split(' ', 1)}
- ""","""
+ """,u"""
['Douglas', 'Noel', ' Adams']
['Douglas', 'Noel Adams']
""")
def test_default_separator(self):
- self.run_test("""
+ self.run_test(u"""
%{'Douglas Noel Adams'|split()}
%{'Douglas Noel Adams'|split(none)}
- ""","""
+ """,u"""
['Douglas', 'Noel', 'Adams']
['Douglas', 'Noel', 'Adams']
""")
def test_max_split_and_default_separator(self):
- self.run_test("""
+ self.run_test(u"""
%{'Douglas Noel Adams'|split(m=2)}
%{'Douglas Noel Adams'|split(m=1)}
- ""","""
+ """,u"""
['Douglas', 'Noel', 'Adams']
['Douglas', 'Noel Adams']
""")
@@ -765,18 +813,18 @@ class SplitTestCase(MkexpSimpleTestCase):
class FilterTestCase(MkexpSimpleTestCase):
def test_basic(self):
- self.run_test("""
+ self.run_test(u"""
%{['Douglas', 'Noel', '', 'Adams']|filter}
- ""","""
+ """,u"""
['Douglas', 'Noel', 'Adams']
""")
class WordwrapTestCase(MkexpSimpleTestCase):
def test_basic(self):
- self.run_test("""
+ self.run_test(u"""
%{'long-arbitrarilyhyphenated textlike-message'|wordwrap(15)}
- """, """
+ """, u"""
long-arbitraril
yhyphenated
textlike-
@@ -784,9 +832,9 @@ class WordwrapTestCase(MkexpSimpleTestCase):
""")
def test_keep_long(self):
- self.run_test("""
+ self.run_test(u"""
%{'long-arbitrarilyhyphenated textlike-message'|wordwrap(15,false)}
- """, """
+ """, u"""
long-
arbitrarilyhyphenated
textlike-
@@ -794,17 +842,17 @@ class WordwrapTestCase(MkexpSimpleTestCase):
""")
def test_keep_long_hyphens(self):
- self.run_test("""
+ self.run_test(u"""
%{'long-arbitrarilyhyphenated textlike-message'|wordwrap(15,false,false)}
- """, """
+ """, u"""
long-arbitrarilyhyphenated
textlike-message
""")
def test_keep_hyphens(self):
- self.run_test("""
+ self.run_test(u"""
%{'long-arbitrarilyhyphenated textlike-message'|wordwrap(15,true,false)}
- """, """
+ """, u"""
long-arbitraril
yhyphenated tex
tlike-message
@@ -813,159 +861,159 @@ class WordwrapTestCase(MkexpSimpleTestCase):
class ListTestCase(MkexpSimpleTestCase):
def test_list_on_string(self):
- self.run_test("""
+ self.run_test(u"""
%{'first'|list}
- """, """
+ """, u"""
['first']
""")
def test_list_on_empty_string(self):
- self.run_test("""
+ self.run_test(u"""
%{''|list}
- """, """
+ """, u"""
[]
""")
def test_list_keep_empty_string(self):
- self.run_test("""
+ self.run_test(u"""
%{''|list(true)}
- """, """
+ """, u"""
['']
""")
def test_list_on_list(self):
- self.run_test("""
+ self.run_test(u"""
%{['first', 'second', 'third']|list}
- """, """
+ """, u"""
['first', 'second', 'third']
""")
def test_list_on_int(self):
- self.run_test("""
+ self.run_test(u"""
%{42|list}
- """, """
+ """, u"""
[42]
""")
def test_list_on_tuple(self):
- self.run_test("""
+ self.run_test(u"""
%{('first', 'second', 'third')|list}
- """, """
+ """, u"""
['first', 'second', 'third']
""")
class JoinTestCase(MkexpSimpleTestCase):
def test_join_on_string(self):
- self.run_test("""
+ self.run_test(u"""
%{'first'|join(', ')}
- """, """
+ """, u"""
first
""")
def test_join_on_empty_string(self):
- self.run_test("""
+ self.run_test(u"""
%{''|join}
- """, """
+ """, u"""
""")
def test_join_on_list(self):
- self.run_test("""
+ self.run_test(u"""
%{['first', 'second', 'third']|join(', ')}
- """, """
+ """, u"""
first, second, third
""")
def test_join_on_int(self):
- self.run_test("""
+ self.run_test(u"""
%{42|join(', ')}
- """, """
+ """, u"""
42
""")
def test_join_on_tuple(self):
- self.run_test("""
+ self.run_test(u"""
%{('first', 'second', 'third')|join(', ')}
- """, """
+ """, u"""
first, second, third
""")
class IsSetTestCase(MkexpSimpleTestCase):
def test_empty_string(self):
- self.run_test("""
+ self.run_test(u"""
%{'' is set}
- """, """
+ """, u"""
False
""")
def test_true(self):
- self.run_test("""
+ self.run_test(u"""
%{'true' is set}
- """, """
+ """, u"""
True
""")
def test_namelist_true(self):
- self.run_test("""
+ self.run_test(u"""
%{'.true.' is set}
- """, """
+ """, u"""
True
""")
def test_false(self):
- self.run_test("""
+ self.run_test(u"""
%{'false' is set}
- """, """
+ """, u"""
False
""")
def test_test(self):
- self.run_test("""
+ self.run_test(u"""
%{'.test.' is set}
- """, """
+ """, u"""
True
""")
def test_undefined(self):
- self.run_test("""
+ self.run_test(u"""
%{undefined_variable_name is set}
- """, """
+ """, u"""
False
""")
def test_undefined_with_true_default(self):
- self.run_test("""
+ self.run_test(u"""
%{undefined_variable_name|d('t') is set}
- """, """
+ """, u"""
True
""")
class FilesTestCase(MkexpSimpleTestCase):
def test_get_file_simple(self):
- self.run_test("""
+ self.run_test(u"""
%{get_file(files, 'target.txt')}
%{get_file(files, 'broken.txt')}
- """, """
+ """, u"""
source.txt
.
- """, """
+ """, u"""
[files]
target.txt = source.txt
broken.txt = .
""")
def test_get_file_path(self):
- self.run_test("""
+ self.run_test(u"""
%{get_file(files, 'target.txt')}
%{get_file(files, 'path.txt')}
%{get_file(files.subdir, 'target.txt')}
- """, """
+ """, u"""
/path/to/source/source.txt
/just/this/one/source.txt
/path/to/source/subdir/source.txt
- """, """
+ """, u"""
[files]
.base_dir = /path/to/source
target.txt = source.txt
@@ -976,15 +1024,17 @@ class FilesTestCase(MkexpSimpleTestCase):
""")
def test_get_file_variable(self):
- self.run_test("""
+ self.run_test(u"""
+ %{JOB.id}
%{get_file(files, 'target.txt')}
%{get_file(files, 'broken.txt')}
%{get_file(files, 'incomplete.txt')}
- """, """
+ """, u"""
+ job
source.txt
$BASENAME.txt
${DOES_NOT_EXIST}.txt
- """, """
+ """, u"""
BASENAME = source
[files]
target.txt = $${BASENAME}.txt
@@ -993,13 +1043,13 @@ class FilesTestCase(MkexpSimpleTestCase):
""")
def test_get_dir(self):
- self.run_test("""
+ self.run_test(u"""
%{get_dir(files)}
%{get_dir(files.subdir)}
- """, """
+ """, u"""
/path/to/source
/path/to/source/subdir
- """, """
+ """, u"""
[files]
.base_dir = /path/to/source
[[subdir]]
@@ -1010,65 +1060,65 @@ class GetTemplatesTestCase(MkexpSimpleTestCase):
def test_by_config_file_name(self):
other_exp_id = 'test_something_completely_different'
- writetemplate(self.exp_id, self.job_id, """
+ writetemplate(self.exp_id, self.job_id, u"""
selected by config file name
""")
self.run_no_template(join(other_exp_id, other_exp_id+'.'+self.job_id),
- """
+ u"""
selected by config file name
- """, """
- EXP_ID = """+other_exp_id+"""
+ """, u"""
+ EXP_ID = """+other_exp_id+u"""
""")
def test_by_exp_id(self):
other_exp_id = 'test_something_completely_different'
- writetemplate(other_exp_id, self.job_id, """
+ writetemplate(other_exp_id, self.job_id, u"""
selected by EXP_ID
""")
self.run_no_template(join(other_exp_id, other_exp_id+'.'+self.job_id),
- """
+ u"""
selected by EXP_ID
- """, """
- EXP_ID = """+other_exp_id+"""
+ """, u"""
+ EXP_ID = """+other_exp_id+u"""
""")
class DelimiterTestCase(MkexpSimpleTestCase):
def test_statement(self):
- self.run_test("""
+ self.run_test(u"""
{%__mkexp__
set x = 'Hello, world!'
%}
%{x}
- """, """
+ """, u"""
Hello, world!
""")
def test_comment(self):
- self.run_test("""
+ self.run_test(u"""
{#__mkexp__
Now you see me - now you don't
#}
- """, """
+ """, u"""
""")
class InheritanceTestCase(MkexpSimpleTestCase):
def test_child_template(self):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
EXP_TYPE =
[jobs]
[[job1]]
[[job2]]
.extends = job1
""")
- writetemplate(self.exp_id, 'job1', """
+ writetemplate(self.exp_id, 'job1', u"""
%{JOB.id} as in job1
""")
- writetemplate(self.exp_id, 'job2', """
+ writetemplate(self.exp_id, 'job2', u"""
%{JOB.id} as in job2
""")
- expected = align("""
+ expected = align(u"""
job2 as in job2
""")
ignore = output(script("mkexp "+self.exp_id+".config"))
@@ -1078,17 +1128,17 @@ class InheritanceTestCase(MkexpSimpleTestCase):
self.assertMultiLineEqual(expected, result)
def test_parent_template(self):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
EXP_TYPE =
[jobs]
[[job1]]
[[job2]]
.extends = job1
""")
- writetemplate(self.exp_id, 'job1', """
+ writetemplate(self.exp_id, 'job1', u"""
%{JOB.id} as in job1
""")
- expected = align("""
+ expected = align(u"""
job2 as in job1
""")
ignore = output(script("mkexp "+self.exp_id+".config"))
@@ -1098,7 +1148,7 @@ class InheritanceTestCase(MkexpSimpleTestCase):
self.assertMultiLineEqual(expected, result)
def test_grandparent_template(self):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
EXP_TYPE =
[jobs]
[[job1]]
@@ -1107,10 +1157,10 @@ class InheritanceTestCase(MkexpSimpleTestCase):
[[job3]]
.extends = job2
""")
- writetemplate(self.exp_id, 'job1', """
+ writetemplate(self.exp_id, 'job1', u"""
%{JOB.id} as in job1
""")
- expected = align("""
+ expected = align(u"""
job3 as in job1
""")
ignore = output(script("mkexp "+self.exp_id+".config"))
@@ -1120,7 +1170,7 @@ class InheritanceTestCase(MkexpSimpleTestCase):
self.assertMultiLineEqual(expected, result)
def test_variable_ancestry(self):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
EXP_TYPE =
[jobs]
var_0 = from jobs
@@ -1137,35 +1187,35 @@ class InheritanceTestCase(MkexpSimpleTestCase):
var_0 = not needed
var_3 = from job3
""")
- writetemplate(self.exp_id, 'job1', """
+ writetemplate(self.exp_id, 'job1', u"""
%{JOB.id}
%{JOB.var_0}
%{JOB.var_1}
%{JOB.var_2}
%{JOB.var_3}
""")
- expecteds = map(align, (
- """
+ expecteds = list(map(align, (
+ u"""
job1
from jobs
from job1
from job1
from job1
""",
- """
+ u"""
job2
from jobs
from job1
from job2
from job2
""",
- """
+ u"""
job3
not needed
from job1
from job2
from job3
- """))
+ """)))
ignore = output(script("mkexp "+self.exp_id+".config"))
for i in (1, 2, 3):
result = readfile(join("test", "experiments", self.exp_id,
@@ -1174,7 +1224,7 @@ class InheritanceTestCase(MkexpSimpleTestCase):
self.assertMultiLineEqual(expecteds[i-1], result)
def test_namelist_override(self):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
EXP_TYPE =
[namelists]
[[namelist]]
@@ -1195,21 +1245,21 @@ class InheritanceTestCase(MkexpSimpleTestCase):
[[[[[group]]]]]
var = 2
""")
- writetemplate(self.exp_id, 'job', """
+ writetemplate(self.exp_id, 'job', u"""
%{NAMELIST}
""")
expecteds = {
- 'job': align("""
+ 'job': align(u"""
&group
var = 999
/
"""),
- 'job1': align("""
+ 'job1': align(u"""
&group
var = 1
/
"""),
- 'job2': align("""
+ 'job2': align(u"""
&group
var = 2
/
@@ -1225,7 +1275,7 @@ class InheritanceTestCase(MkexpSimpleTestCase):
class JobSiblingsTestCase(MkexpSimpleTestCase):
def test_sibling_lookup(self):
- writeconfig(self.exp_id, """
+ writeconfig(self.exp_id, u"""
EXP_TYPE =
[jobs]
[[job1]]
@@ -1233,17 +1283,17 @@ class JobSiblingsTestCase(MkexpSimpleTestCase):
[[job2]]
seniority = younger
""")
- writetemplate(self.exp_id, 'job1', """
+ writetemplate(self.exp_id, 'job1', u"""
%{JOB.id}: %{JOB.seniority}
%{jobs.job1.id}: %{jobs.job1.seniority}
%{jobs.job2.id}: %{jobs.job2.seniority}
""")
- writetemplate(self.exp_id, 'job2', """
+ writetemplate(self.exp_id, 'job2', u"""
%{JOB.id}: %{JOB.seniority}
%{jobs.job2.id}: %{jobs.job2.seniority}
%{jobs.job1.id}: %{jobs.job1.seniority}
""")
- expected = align("""
+ expected = align(u"""
job1: elder
job1: elder
job2: younger
@@ -1264,7 +1314,7 @@ class NativeVariableTestCase(MkexpSimpleTestCase):
def test_var_statement(self):
exp_id = "test_var_statement"
job_id = "job"
- writeconfig(exp_id, """
+ writeconfig(exp_id, u"""
EXP_TYPE =
GLOBAL1 = 123$${VAR1}456
GLOBAL2 = $${VAR2}$${VAR3}
@@ -1275,17 +1325,17 @@ class NativeVariableTestCase(MkexpSimpleTestCase):
[[[group]]]
key = abc$${var}def
[jobs]
- [["""+job_id+"""]]
+ [["""+job_id+u"""]]
.var_format = <<<%s>>>
""")
- writetemplate(exp_id, job_id, """
+ writetemplate(exp_id, job_id, u"""
GLOBAL1=%{GLOBAL1}
GLOBAL2=%{GLOBAL2}
GLOBAL3='%{GLOBAL3|join(" ")}'
GLOBAL4=%{context("GLOBAL<<<FOUR>>>")}
%{NAMELIST}
""")
- expected = align("""
+ expected = align(u"""
GLOBAL1=123<<<VAR1>>>456
GLOBAL2=<<<VAR2>>><<<VAR3>>>
GLOBAL3='1 <<<VAR2>>> 3'
@@ -1301,7 +1351,7 @@ class NativeVariableTestCase(MkexpSimpleTestCase):
def test_var_separation(self):
exp_id = "test_var_separation"
- writeconfig(exp_id, """
+ writeconfig(exp_id, u"""
EXP_TYPE =
[jobs]
use_native_var = $${native_var}
@@ -1309,16 +1359,16 @@ class NativeVariableTestCase(MkexpSimpleTestCase):
.var_format = <<<%s>>>
[[job2]]
""")
- writetemplate(exp_id, "job1", """
+ writetemplate(exp_id, "job1", u"""
%{JOB.use_native_var}
""")
- writetemplate(exp_id, "job2", """
+ writetemplate(exp_id, "job2", u"""
%{JOB.use_native_var}
""")
- expected1 = align("""
+ expected1 = align(u"""
<<<native_var>>>
""")
- expected2 = align("""
+ expected2 = align(u"""
${native_var}
""")
ignore = output(script("mkexp "+exp_id+".config"))
@@ -1329,5 +1379,17 @@ class NativeVariableTestCase(MkexpSimpleTestCase):
self.assertMultiLineEqual(expected1, result1)
self.assertMultiLineEqual(expected2, result2)
+class UnicodeTestCase(MkexpSimpleTestCase):
+
+ def test_value(self):
+ self.run_test(u"""
+ %{VAR}
+ """, u"""
+ ÄÖÜäöüß😉
+ """, u"""
+ VAR = ÄÖÜäöüß😉
+ """)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/test0001.config b/test/test0001.config
index c616561c490eb9dff0ac0bd0e32beaf8f780aa38..b78434dddd364d588086e8f2d161ba3f76d8467b 100644
--- a/test/test0001.config
+++ b/test/test0001.config
@@ -1,3 +1,5 @@
+# Basic unicode check: ÄÖÜäöüß😉
+
EXP_TYPE = DEFAULT
VAR1 = value1