mkexp: added 'getconfig' tool to generate clean config files from 'update' scripts

CHANGES.txt

@@ -9,6 +9,11 @@ Release Changes
 Release 1.0.5
 =============
 
+Global
+------
+
+* Added 'getconfig' tool to generate clean config files from 'update' scripts
+
 Templates
 ---------
 

expargparse.py

+'''
+Specialized argument parser for mkexp tool.
+
+$Id: expargparse -1 2019-06-27 07:00:00Z m221078 $
+'''
+
+import argparse
+
+import package_info
+
+class MkexpArgumentParser(argparse.ArgumentParser):
+    ''' Specialized argument parser for mkexp tool
+    
+    Also used for parsing the 'update' script provided with the run scripts
+    '''
+    def __init__(self):
+        argparse.ArgumentParser.__init__(self, description=
+            'Generate an experiment from templates and the given configuration file.')
+        self.add_argument('config', help='experiment configuration file name')
+        self.add_argument('assigns', metavar='key=value', nargs='*',
+                           help='override configuration file settings')
+        self.add_argument('-V', '--version', action='version',
+                          version=package_info.version)
+        self.add_argument('-p', '--path', 
+                          help='search path for default config and templates')
+        self.add_argument('-m', '--no-make-dirs',
+                          action='store_false', dest='make_dirs',
+                          help='do not create work and data directories')
+        self.add_argument('-q', '--quiet',
+                          action='store_true', dest='quiet',
+                          help='suppress informative messages')
+        self.add_argument('-g', '--getexp', action='store_true',
+                          help='load flat config (from getexp -vv)') 
+
+def assigns_to_dicts(args):
+    
+    def value_to_list(value):
+        result = value.split(',')
+        if len(result) == 1:
+            result = result[0]
+        return result
+
+    def assign_to_dict(assign):
+        (key, value) = assign.split('=', 1)
+        current = value_to_list(value)
+        pos = key.rfind('.')
+        while(pos >= 0):
+            if key[pos-1] == '.':
+                key = key[0:pos-1]+key[pos:]
+                pos = key.rfind('.', 0, pos-1)
+            else:
+                current = {key[pos+1:]: current}
+                key = key[0:pos]
+                pos = key.rfind('.')
+        if key:
+            current = {key: current}
+        return current
+
+    return map(assign_to_dict, args.assigns)
+

getconfig

+#! /usr/bin/env python
+#
+# Reconstruct config from update file, including command line
+#
+# $Id: getconfig -1 2019-06-26 18:00:00Z m221078 $
+#
+
+import argparse
+import os
+import re
+import sys
+
+from configobj import ConfigObj
+
+import expargparse
+from feedback import die
+import package_info
+
+#
+# Main routine
+#
+
+# Check command line
+
+command_line = argparse.ArgumentParser(description=
+    'Reconstruct config from update file, including command line.')
+command_line.add_argument('update', nargs='?', default='update', 
+                          help='update script file name [%(default)s]')
+command_line.add_argument('-V', '--version', action='version',
+                          version=package_info.version)
+
+args = command_line.parse_args()
+
+# Check mkexp command line
+
+mkexp_line = expargparse.MkexpArgumentParser() 
+
+# Experiment configuration
+
+config_dir = ''
+config_file = ''
+config_dicts = []
+
+try:
+    update_file = open(args.update)
+
+    for line in update_file:
+        match = re.match(r"^cd '(.*)'$", line)
+        if match:
+            config_dir = match.group(1)
+        else:
+            match = re.match(r"^exec '.*?' '(.*)'", line)
+            if match:
+                mkexp_args = mkexp_line.parse_args(match.group(1).split("' '"))
+                config_file = mkexp_args.config
+                config_dicts = expargparse.assigns_to_dicts(mkexp_args)
+
+    update_file.close()
+except IOError as error:
+    die("'{0}': {1}".format(error.filename, error.strerror))
+
+if not os.path.isabs(config_file):
+    config_file = os.path.join(config_dir, config_file)
+
+config_data = ConfigObj(config_file, interpolation=False)
+for d in config_dicts:
+    config_data.merge(d)
+
+config_data.indent_type = '  '
+config_data.write(sys.stdout)
+

mkexp

@@ -5,7 +5,6 @@
 # $Id$
 #
 
-import argparse
 import codecs
 import os
 import re
@@ -20,6 +19,7 @@ import jinja2
 from jinja2 import Environment, ChoiceLoader, FileSystemLoader, \
                    TemplateNotFound, TemplatesNotFound, is_undefined
 
+import expargparse
 from expconfig import ExpConfig, ExpConfigError
 import feedback
 import files
@@ -220,30 +220,6 @@ def format_vars(section, key, log, fmt):
     elif '__iter__' in dir(value) and not isinstance(value, dict):
         # Format all list elements
         section[newkey] = map(transform, value)
- 
-# Command line parsing
-
-def value_to_list(value):
-    result = value.split(',')
-    if len(result) == 1:
-        result = result[0]
-    return result
-
-def assign_to_dict(assign):
-    (key, value) = assign.split('=', 1)
-    current = value_to_list(value)
-    pos = key.rfind('.')
-    while(pos >= 0):
-        if key[pos-1] == '.':
-            key = key[0:pos-1]+key[pos:]
-            pos = key.rfind('.', 0, pos-1)
-        else:
-            current = {key[pos+1:]: current}
-            key = key[0:pos]
-            pos = key.rfind('.')
-    if key:
-        current = {key: current}
-    return current
 
 #
 # Main routine
@@ -259,22 +235,7 @@ config_roots = os.environ.get('MKEXP_PATH', '').split(':')
 
 # Check command line
 
-command_line = argparse.ArgumentParser(description=
-    'Generate an experiment from templates and the given configuration file.',
-    version=package_info.version)
-command_line.add_argument('config', help='experiment configuration file name')
-command_line.add_argument('assigns', metavar='key=value', nargs='*',
-                           help='override configuration file settings')
-command_line.add_argument('--path', '-p', 
-                          help='search path for default config and templates')
-command_line.add_argument('--no-make-dirs', '-m',
-                          action='store_false', dest='make_dirs',
-                          help='do not create work and data directories')
-command_line.add_argument('--quiet', '-q',
-                          action='store_true', dest='quiet',
-                          help='suppress informative messages')
-command_line.add_argument('--getexp', '-g', action='store_true',
-                          help='load flat config (from getexp -vv)') 
+command_line = expargparse.MkexpArgumentParser()
 
 args = command_line.parse_args()
 
@@ -385,8 +346,8 @@ template_env.tests['set'] = is_set
 # Store environment as default for control settings, then add config from files
 
 extra_dict = ConfigObj(interpolation=False)
-for assign in args.assigns:
-    extra_dict.merge(assign_to_dict(assign))
+for assign_dict in expargparse.assigns_to_dicts(args):
+    extra_dict.merge(assign_dict)
 extra_dict['mkexp_input'] = 'Generated by $Id$'
 if not args.getexp:
     extra_dict['mkexp_input'] = extra_dict['mkexp_input'].replace('$', '$$')

test.py

@@ -180,6 +180,14 @@ class CommandLineTestCase(MkexpTestCase):
         """)
         self.assertMultiLineEqual(expected, result)
 
+    def test_getconfig(self):
+        ignore = output(script("""
+            mkexp test0001.config VAR4=value4 jobs.run.time_limit=12:34:56
+        """))
+        result = output(script('getconfig experiments/test0001/update'))
+        self.assertIn('VAR4 = value4', result)
+        self.assertIn('time_limit = 12:34:56', result)
+
 class ContentTestCase(MkexpSimpleTestCase):
 
     def test_job_override(self):