Added 'selconfig' and 'files2config' tools, updated 'namelist2config'

CHANGES.txt

@@ -9,6 +9,14 @@ Release Changes
 Milestone 1.1.0
 ===============
 
+Global
+------
+
+* TODO: Porting to python3
+* Added 'selconfig' tool to extract sections from config files
+* Added 'files2config' tool to extract input file info from logs and scripts
+* Added variable replacement beta options to the '...2config' tools
+
 Configuration
 -------------
 
@@ -22,11 +30,6 @@ Templates
 * Added optional parameter 'default_value' to 'format_namelists' to define an
   alternative default value for namelist variables
 
-Global
-------
-
-* TODO: Porting to python3
-
 Release 1.0.12
 ==============
 

files2config

+#! /usr/bin/env perl
+
+use strict;
+use warnings;
+
+use Getopt::Std;
+$Getopt::Std::STANDARD_HELP_VERSION = 1;
+
+my %subst_map = (
+    i => 'EXP_ID',
+    a => 'ATMO_GRID_ID',
+    o => 'OCEAN_GRID_ID',
+    A => 'ATMO_GRID_TYPE',
+    O => 'OCEAN_GRID_TYPE',
+);
+
+my $direct = 0;
+my $xlog = '';
+my %opts = ();
+for my $var (keys %subst_map) {
+    $opts{$var} = '';
+}
+
+getopts('dx'.join(':', keys(%subst_map), ''), \%opts) or die("\n");
+exists $opts{d} and $direct = 1;
+exists $opts{x} and $xlog = '\+';
+
+my %file_info = ();
+
+while(<>) {
+    if(my ($mtd, undef, $src, $dst) =
+       m{^$xlog\s*((cdo|cp|ln)\s.*)\s+(\S+)\s+(\S+)\s*$}) {
+         #DEBUG# warn($_, join(':', $mtd, $src, $dst), "\n"); #DEBUG#
+         # Clean source file name
+         $src =~ s{/+$}{};
+         $src =~ s{^\./+}{};
+         # Clean destination file name
+         $dst =~ s{/+$}{};
+         $dst =~ s{^\./+}{};
+         my $dir = $src;
+         if($dir =~ m{/}) {
+             $dir =~ s{^(.*)/.*$}{$1};
+             $src =~ s{^.*/}{};
+         }
+         else {
+             $dir = '.';
+         }
+         unless($dst && $dst ne '.') {
+             $dst = $src
+         }
+         $direct and print($_, join(':', $mtd, $dir, $src, $dst), "\n");
+         my $key = "$dir:$src:$dst";
+         exists $file_info{$key} and die("duplicate key '$key'\n");
+         $file_info{$key} = [$mtd, $dir, $src, $dst];
+    }
+}
+
+sub subst_key($) {
+    my $key = shift;
+    for my $var (keys %subst_map) {
+        if($opts{$var}) {
+            $key =~ s/[._]$opts{$var}//;
+        }
+    }
+    return $key;
+}
+
+sub subst_value($) {
+    my $value = shift;
+    for my $var (keys %subst_map) {
+        if($opts{$var}) {
+            $value =~ s/$opts{$var}(\w)/\${$subst_map{$var}}$1/;
+            $value =~ s/$opts{$var}/\$$subst_map{$var}/;
+        }
+    }
+    return $value;
+}
+
+unless($direct) {
+    my $i = 0;
+    print("[files]\n");
+    my $last_mtd = '';
+    my $last_dir = '';
+    for my $key (sort(keys %file_info)) {
+        my ($mtd, $dir, $src, $dst) = @{$file_info{$key}};
+        if($last_dir ne $dir || $last_mtd ne $mtd) {
+            printf("  [[section_%02d]]\n", ++$i);
+            print("    .base_dir = $dir\n");
+            print("    .method = $mtd\n");
+        }
+        if($last_dir ne $dir) {
+            $last_dir = $dir;
+        }
+        if($last_mtd ne $mtd) {
+            $last_mtd = $mtd;
+        }
+        $dst = subst_key($dst);
+        $src = subst_value($src);
+        print("    $dst = $src\n");
+        ### print(join(' ', @{$file_info{$key}}), "\n");
+    }
+}

namelist2config

@@ -36,11 +36,45 @@ sub format_value($) {
     return $value;
 }
 
+my %subst_map = (
+    i => 'EXP_ID',
+    a => 'ATMO_GRID_ID',
+    o => 'OCEAN_GRID_ID',
+    A => 'ATMO_GRID_TYPE',
+    O => 'OCEAN_GRID_TYPE',
+);
+
+my %opts = ();
+
+sub subst_key($) {
+    my $key = shift;
+    for my $var (keys %subst_map) {
+        if($opts{$var}) {
+            $key =~ s/[._]$opts{$var}//;
+        }
+    }
+    return $key;
+}
+
+sub subst_value($) {
+    my $value = shift;
+    for my $var (keys %subst_map) {
+        if($opts{$var}) {
+            $value =~ s/$opts{$var}(\w)/\${$subst_map{$var}}$1/;
+            $value =~ s/$opts{$var}/\$$subst_map{$var}/;
+        }
+    }
+    return $value;
+}
+
 my $verbose = 0;
 my $direct = 0;
 my $comments = 0;
-my %opts;
-getopts('dvc', \%opts);
+for my $var (keys %subst_map) {
+    $opts{$var} = '';
+}
+
+getopts('dvc'.join(':', keys(%subst_map), ''), \%opts) or die("\n");
 exists $opts{v} and $verbose = 1;
 exists $opts{d} and $direct = 1;
 exists $opts{c} and $comments = 1;
@@ -74,8 +108,9 @@ while(<>) {
     elsif(/(\S+)\s*0?<<\s*\\?EOF/) {
         $in_doc = 1;
         $direct and $verbose and print "### $_";
-        exists $namelists{$1} or $namelists{$1} = {};
-        $namelist_file = $namelists{$1};
+        my $key = subst_key($1);
+        exists $namelists{$key} or $namelists{$key} = {};
+        $namelist_file = $namelists{$key};
     }
     ### elsif($in_doc && m{^\s*&(\w+)}) {
     elsif(m{^\s*&(\w+)}) {

selconfig

+#! /usr/bin/env python
+'''\
+Select the given section of a config file
+
+$Id$
+'''
+
+import argparse
+import re
+import sys
+import StringIO
+
+from configobj import ConfigObj
+
+from expargparse import assigns_to_dicts, get_key_chain
+from feedback import die
+import package_info
+
+#
+# Main routine
+#
+
+# Check command line
+
+command_line = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0])
+command_line.add_argument('section',
+                          help='section to be selected, in . notation')
+command_line.add_argument('config', nargs='?', default='-', 
+                          help='original configuration file name [%(default)s]')
+command_line.add_argument('-V', '--version', action='version',
+                          version=package_info.version)
+command_line.add_argument('--inline-comments' , '-c', action='store_true',
+                          help='compact white space before inline comments'
+                               ' (BETA)')
+command_line.add_argument('--trailing-space' , '-t', action='store_true',
+                          help='remove white space at end of lines')
+
+args = command_line.parse_args()
+
+# File handling
+
+try:
+    config_file = args.config
+    if config_file == '-':
+        config_file = sys.stdin
+    config_data = ConfigObj(config_file, interpolation=False, file_error=True)
+except IOError as error:
+    die(error.message)
+
+# Walk config to the appropriate section and create output structure
+
+selected_data = ConfigObj(interpolation=False, write_empty_values=True,
+                          indent_type='  ')
+config = config_data
+selected = selected_data
+for section in reversed(get_key_chain(args.section)):
+    if section in config.comments:
+        selected.comments[section] = config.comments[section]
+    if section in config.inline_comments:
+        selected.inline_comments[section] = config.inline_comments[section]
+    config = config[section]
+    selected[section] = {}
+    selected = selected[section]
+# Replace the empty leaf section by the original section to be selected
+selected.parent[section] = config
+
+# Ready to roll out
+
+lines = StringIO.StringIO()
+selected_data.write(lines)
+
+lines.seek(0)
+for line in lines:
+    if args.inline_comments: line = re.sub(r' = (.*?)  #', r' = \1 #', line)
+    if args.trailing_space:
+        print(line.rstrip())
+    else:
+        sys.stdout.write(line)

setup.py

@@ -10,7 +10,7 @@ setup(
     author_email = 'karl-hermann.wieners@mpimet.mpg.de',
     url = 'http://code.mpimet.mpg.de/projects/esmenv',
     py_modules = ['_version', 'configobj', 'validate', 'feedback', 'expargparse', 'expconfig', 'files', 'package_info', 'update'],
-    scripts = ['mkexp', 'getexp', 'rmexp', 'diffexp', 'diffpath', 'cpexp', 'cppath', 'duexp', 'getconfig', 'editexp', 'upexp', 'setconfig', 'namelist2config'],
+    scripts = ['mkexp', 'getexp', 'rmexp', 'diffexp', 'diffpath', 'cpexp', 'cppath', 'duexp', 'getconfig', 'editexp', 'upexp', 'setconfig', 'selconfig', 'namelist2config', 'files2config'],
     data_files = [('share/doc/'+package_info.name, ['doc/mkexp.pdf', 'mkexp.bash'])],
     platforms = ['Posix'],
     license = 'LICENSE.txt',