From 236e0054aa6c4f4e247b01a2d2b4ff5da3d037c8 Mon Sep 17 00:00:00 2001
From: Karl-Hermann Wieners <karl-hermann.wieners@mpimet.mpg.de>
Date: Thu, 16 Jun 2022 09:38:27 +0200
Subject: [PATCH] mkexp/expconfig: replace deep copy of Sections by ordered
dict copy
Fixes long run times for extended jobs/namelist groups
---
CHANGES.txt | 1 +
expconfig.py | 24 ++++++++++++++++++++++++
mkexp | 11 ++++++-----
3 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 89b0edd..001d56d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -20,6 +20,7 @@ Tools
-----
* importexp: fixed extraction of start/end date in input scripts
+* mkexp: fixed long run times for extended jobs/namelist groups
Release 1.1.4
=============
diff --git a/expconfig.py b/expconfig.py
index 419df57..977b5fc 100644
--- a/expconfig.py
+++ b/expconfig.py
@@ -4,6 +4,7 @@ Generate an earth system model configuration from the given configuration file.
$Id$
'''
+import collections
import io
import locale
import os
@@ -36,6 +37,29 @@ def is_not_empty(arg):
else:
return [_f for _f in [x.rstrip() for x in arg] if _f]
+def odict(self):
+ '''Return a deepcopy of self as an ordered dictionary.
+
+ >>> n = odict(a)
+ >>> n == a
+ 1
+ >>> n is a
+ 0
+ '''
+ newdict = collections.OrderedDict()
+ for entry in self:
+ this_entry = self[entry]
+ if isinstance(this_entry, configobj.Section):
+ this_entry = odict(this_entry)
+ elif isinstance(this_entry, list):
+ # create a copy rather than a reference
+ this_entry = list(this_entry)
+ elif isinstance(this_entry, tuple):
+ # create a copy rather than a reference
+ this_entry = tuple(this_entry)
+ newdict[entry] = this_entry
+ return newdict
+
def merge_comments(this, indict):
'''Merge comments from indict into current configuration.
diff --git a/mkexp b/mkexp
index 64de09a..04d666f 100755
--- a/mkexp
+++ b/mkexp
@@ -7,7 +7,6 @@
from __future__ import print_function
-import copy
import io
import os
import re
@@ -500,14 +499,15 @@ def extend(subjob, jobs_config, extended_jobs, job_dict=None):
# Add parent config
if extended_job:
- pre_config.merge(copy.deepcopy(jobs_config[extended_job]))
+ pre_config.merge(expconfig.odict(jobs_config[extended_job]))
+ expconfig.merge_comments(pre_config, jobs_config[extended_job])
# Add actual subjob config
pre_config.merge(subconfig)
# Replace subjob config by extended config
jobs_config[subjob] = {}
- jobs_config[subjob].merge(pre_config)
+ jobs_config[subjob].merge(expconfig.odict(pre_config))
expconfig.merge_comments(jobs_config[subjob], pre_config)
del pre_config
@@ -564,8 +564,9 @@ for subjob, subconfig in jobs_config.items():
# Prepare namelists for inclusion in scripts
var_format = job_config['JOB'].get('.var_format', '${%s}')
var_list = set()
- del job_config['jobs']
- job_config['jobs'] = copy.deepcopy(jobs_config)
+ job_config['jobs'] = {}
+ job_config['jobs'].merge(expconfig.odict(jobs_config))
+ expconfig.merge_comments(job_config['jobs'], jobs_config)
job_config.walk(format_vars, log=var_list, fmt=var_format)
job_config['VARIABLES_'] = var_list
for namelist, groups in job_config['namelists'].items():
--
GitLab