Commit a628a186 authored by Sebastian Milinski's avatar Sebastian Milinski
Browse files

Adding/updating scripts to extract 6-hourly output for various variables

parent 5b1b6eb5
#!/bin/bash
#
# based on a script by Frank Sienz
#
# requires: module load gnu-parallel on mistral
module load gnu-parallel
module load cdo
#
# approx. time duration for one ensemble member
# (years 1850-1999, 6h output, on mistralpp, calc 500hPA with cdo after):
# number of parallel processes: 1 2 3 4 5 ... 8
# duration (minutes): 295 150 101 75 65 ... 42
# for 10 years: 19.7 10.0 6.7 5.0 4.3 ... 2.8
#
# ! if IO is the bottleneck, parallelisation slows things down
# ! recommandation for other uses: benchmark test runs with different number of cores
# number of parallel processes
#
ncore=10
# 6h output: member 69 to 100
#
ensfirst=69
enslast=100
# years: 1850 to 1999
#
yrfirst=1850
yrlast=1999
#
#
direxp=/work/mh1007/MPI-GE/onepct
dirout=/work/mh0033/m300265/processed_thesis/onepct/hf_output
dirtmp=$SCRATCH/onepct
mkdir -p $direxp
mkdir -p $dirtmp
var=evap
dir_final=/work/mh1007/MPI-GE_processed/onepct/hf_output/${var}
mkdir -p $dir_final
###
#
# namelist for cdo after (cdo manual)
cat <<EOF > namelist_${var}.txt
TYPE=20
CODE=182
LEVEL=1
INTERVAL=0
MEAN=0
EOF
###
#
# function for processing
# requires exported variables: dirin, dirout, ensdesc, var
processvar() {
year=$1
fin=${dirin}/onepct${ensdesc}_echam6_echam_${year}.grb
fout=${dirtmp}/onepct-${var}_${ensdesc}_${year}.grb
echo -ne "* ${year} --- `date +%T` --- ${fin} "'\r'
if [ -e ${fin} ]; then
#
cdo -s after ${fin} ${fout} <namelist_${var}.txt
if [ $? -ne 0 ]; then
echo
echo -n "error for file: "
echo ${fin}
exit 1
fi
else
echo
echo -n "missing file: "
echo ${fin}
exit 1
fi
}
export -f processvar
###
#
for ensdesc in $(seq -f "%04g" ${ensfirst} 1 ${enslast}); do
echo
echo "*************************************************************************************************"
echo "* `date +%T` --- ensembles (${ensfirst}-${enslast}): ${ensdesc} --- ${yrfirst} to ${yrlast}"
echo "*"
dirin=${direxp}/onepct${ensdesc}/outdata/echam6
#
#
tstart=$(date +%s)
# export required vars
#
export dirin
export dirout
export dirtmp
export ensdesc
export var
# error -> stop spawning new jobs: --halt soon,fail=1
# "sorted" output: -k
#
parallel -k -j ${ncore} --halt soon,fail=1 processvar ::: `seq ${yrfirst} 1 ${yrlast}`
#
tend=$(date +%s)
tdiff=$(echo "scale=2; ($tend - $tstart)/60" | bc)
echo "Merge years into single file..."
ifiles=${dirtmp}/onepct-${var}_${ensdesc}_????.grb
ofile=${dir_final}/onepct_${yrfirst}-${yrlast}_ens_${ensdesc}.${var}.nc
cdo -f nc -t echam6 mergetime $ifiles $ofile
echo
echo "created $ofile"
echo "*"
echo "* `date +%T` --- ${tdiff}m"
echo "*"
done
...@@ -22,8 +22,8 @@ ncore=10 ...@@ -22,8 +22,8 @@ ncore=10
# 6h output: member 69 to 100 # 6h output: member 69 to 100
# #
ensfirst=69 ensfirst=72 #69
enslast=100 enslast=72 #100
# years: 1850 to 1999 # years: 1850 to 1999
# #
......
...@@ -22,8 +22,8 @@ ncore=10 ...@@ -22,8 +22,8 @@ ncore=10
# 6h output: member 69 to 100 # 6h output: member 69 to 100
# #
ensfirst=69 ensfirst=72 #69
enslast=100 enslast=72 #100
# years: 1850 to 1999 # years: 1850 to 1999
# #
......
#!/bin/bash
#
# based on a script by Frank Sienz
#
# requires: module load gnu-parallel on mistral
module load gnu-parallel
module load cdo
#
# approx. time duration for one ensemble member
# (years 1850-1999, 6h output, on mistralpp, calc 500hPA with cdo after):
# number of parallel processes: 1 2 3 4 5 ... 8
# duration (minutes): 295 150 101 75 65 ... 42
# for 10 years: 19.7 10.0 6.7 5.0 4.3 ... 2.8
#
# ! if IO is the bottleneck, parallelisation slows things down
# ! recommandation for other uses: benchmark test runs with different number of cores
# number of parallel processes
#
ncore=10
# 6h output: member 69 to 100
#
ensfirst=69
enslast=100
# years: 1850 to 1999
#
yrfirst=1850
yrlast=1999
#
#
direxp=/work/mh1007/MPI-GE/onepct
dirout=/work/mh0033/m300265/processed_thesis/onepct/hf_output
dirtmp=$SCRATCH/onepct
mkdir -p $direxp
mkdir -p $dirtmp
var=layer_moisture
dir_final=/work/mh1007/MPI-GE_processed/onepct/hf_output/${var}
mkdir -p $dir_final
###
#
# namelist for cdo after (cdo manual)
# cat <<EOF > namelist_${var}.txt
# TYPE=20
# CODE=151
# LEVEL=1
# INTERVAL=0
# MEAN=0
# EOF
###
#
# function for processing
# requires exported variables: dirin, dirout, ensdesc, var
processvar() {
year=$1
fin=${dirin}/onepct${ensdesc}_jsbach_jsbach_${year}.grb
fout=${dirtmp}/onepct-${var}_${ensdesc}_${year}.grb
echo -ne "* ${year} --- `date +%T` --- ${fin} "'\r'
if [ -e ${fin} ]; then
#
#cdo -s after ${fin} ${fout} <namelist_${var}.txt
cdo selcode,84 ${fin} ${fout}
if [ $? -ne 0 ]; then
echo
echo -n "error for file: "
echo ${fin}
exit 1
fi
else
echo
echo -n "missing file: "
echo ${fin}
exit 1
fi
}
export -f processvar
###
#
for ensdesc in $(seq -f "%04g" ${ensfirst} 1 ${enslast}); do
echo
echo "*************************************************************************************************"
echo "* `date +%T` --- ensembles (${ensfirst}-${enslast}): ${ensdesc} --- ${yrfirst} to ${yrlast}"
echo "*"
dirin=${direxp}/onepct${ensdesc}/outdata/jsbach
#
#
tstart=$(date +%s)
# export required vars
#
export dirin
export dirout
export dirtmp
export ensdesc
export var
# error -> stop spawning new jobs: --halt soon,fail=1
# "sorted" output: -k
#
parallel -k -j ${ncore} --halt soon,fail=1 processvar ::: `seq ${yrfirst} 1 ${yrlast}`
#
tend=$(date +%s)
tdiff=$(echo "scale=2; ($tend - $tstart)/60" | bc)
echo "Merge years into single file..."
ifiles=${dirtmp}/onepct-${var}_${ensdesc}_????.grb
ofile=${dir_final}/onepct_${yrfirst}-${yrlast}_ens_${ensdesc}.${var}.nc
cdo -f nc mergetime $ifiles $ofile
echo
echo "created $ofile"
echo "*"
echo "* `date +%T` --- ${tdiff}m"
echo "*"
done
#!/bin/bash
#
# based on a script by Frank Sienz
#
# requires: module load gnu-parallel on mistral
module load gnu-parallel
module load cdo
#
# approx. time duration for one ensemble member
# (years 1850-1999, 6h output, on mistralpp, calc 500hPA with cdo after):
# number of parallel processes: 1 2 3 4 5 ... 8
# duration (minutes): 295 150 101 75 65 ... 42
# for 10 years: 19.7 10.0 6.7 5.0 4.3 ... 2.8
#
# ! if IO is the bottleneck, parallelisation slows things down
# ! recommandation for other uses: benchmark test runs with different number of cores
# number of parallel processes
#
ncore=10
# 6h output: member 69 to 100
#
ensfirst=69
enslast=100
# years: 1850 to 1999
#
yrfirst=1850
yrlast=1999
#
#
direxp=/work/mh1007/MPI-GE/onepct
dirout=/work/mh0033/m300265/processed_thesis/onepct/hf_output
dirtmp=$SCRATCH/onepct
mkdir -p $direxp
mkdir -p $dirtmp
var=ws
dir_final=/work/mh1007/MPI-GE_processed/onepct/hf_output/${var}
mkdir -p $dir_final
###
#
# namelist for cdo after (cdo manual)
cat <<EOF > namelist_${var}.txt
TYPE=20
CODE=140
LEVEL=1
INTERVAL=0
MEAN=0
EOF
###
#
# function for processing
# requires exported variables: dirin, dirout, ensdesc, var
processvar() {
year=$1
fin=${dirin}/onepct${ensdesc}_echam6_echam_${year}.grb
fout=${dirtmp}/onepct-${var}_${ensdesc}_${year}.grb
echo -ne "* ${year} --- `date +%T` --- ${fin} "'\r'
if [ -e ${fin} ]; then
#
cdo -s after ${fin} ${fout} <namelist_${var}.txt
if [ $? -ne 0 ]; then
echo
echo -n "error for file: "
echo ${fin}
exit 1
fi
else
echo
echo -n "missing file: "
echo ${fin}
exit 1
fi
}
export -f processvar
###
#
for ensdesc in $(seq -f "%04g" ${ensfirst} 1 ${enslast}); do
echo
echo "*************************************************************************************************"
echo "* `date +%T` --- ensembles (${ensfirst}-${enslast}): ${ensdesc} --- ${yrfirst} to ${yrlast}"
echo "*"
dirin=${direxp}/onepct${ensdesc}/outdata/echam6
#
#
tstart=$(date +%s)
# export required vars
#
export dirin
export dirout
export dirtmp
export ensdesc
export var
# error -> stop spawning new jobs: --halt soon,fail=1
# "sorted" output: -k
#
parallel -k -j ${ncore} --halt soon,fail=1 processvar ::: `seq ${yrfirst} 1 ${yrlast}`
#
tend=$(date +%s)
tdiff=$(echo "scale=2; ($tend - $tstart)/60" | bc)
echo "Merge years into single file..."
ifiles=${dirtmp}/onepct-${var}_${ensdesc}_????.grb
ofile=${dir_final}/onepct_${yrfirst}-${yrlast}_ens_${ensdesc}.${var}.nc
cdo -f nc -t echam6 mergetime $ifiles $ofile
echo
echo "created $ofile"
echo "*"
echo "* `date +%T` --- ${tdiff}m"
echo "*"
done
#!/bin/bash
#
# based on a script by Frank Sienz
#
# requires: module load gnu-parallel on mistral
module load gnu-parallel
module load cdo
#
# approx. time duration for one ensemble member
# (years 1850-1999, 6h output, on mistralpp, calc 500hPA with cdo after):
# number of parallel processes: 1 2 3 4 5 ... 8
# duration (minutes): 295 150 101 75 65 ... 42
# for 10 years: 19.7 10.0 6.7 5.0 4.3 ... 2.8
#
# ! if IO is the bottleneck, parallelisation slows things down
# ! recommandation for other uses: benchmark test runs with different number of cores
# number of parallel processes
#
ncore=10
# 6h output: member 69 to 100
#
ensfirst=77
enslast=77
# years: 1850 to 1999
#
yrfirst=1850
yrlast=1999
#
#
direxp=/work/mh1007/MPI-GE/onepct
dirout=/work/mh0033/m300265/processed_thesis/onepct/hf_output
dirtmp=$SCRATCH/onepct
mkdir -p $direxp
mkdir -p $dirtmp
var=u10
dir_final=/work/mh1007/MPI-GE_processed/onepct/hf_output/${var}
mkdir -p $dir_final
###
#
# namelist for cdo after (cdo manual)
cat <<EOF > namelist_${var}.txt
TYPE=20
CODE=165
LEVEL=1
INTERVAL=0
MEAN=0
EOF
###
#
# function for processing
# requires exported variables: dirin, dirout, ensdesc, var
processvar() {
year=$1
fin=${dirin}/onepct${ensdesc}_echam6_echam_${year}.grb
fout=${dirtmp}/onepct-${var}_${ensdesc}_${year}.grb
echo -ne "* ${year} --- `date +%T` --- ${fin} "'\r'
if [ -e ${fin} ]; then
#
cdo -s after ${fin} ${fout} <namelist_${var}.txt
if [ $? -ne 0 ]; then
echo
echo -n "error for file: "
echo ${fin}
exit 1
fi
else
echo
echo -n "missing file: "
echo ${fin}
exit 1
fi
}
export -f processvar
###
#
for ensdesc in $(seq -f "%04g" ${ensfirst} 1 ${enslast}); do
echo
echo "*************************************************************************************************"
echo "* `date +%T` --- ensembles (${ensfirst}-${enslast}): ${ensdesc} --- ${yrfirst} to ${yrlast}"
echo "*"
dirin=${direxp}/onepct${ensdesc}/outdata/echam6
#
#
tstart=$(date +%s)
# export required vars
#
export dirin
export dirout
export dirtmp
export ensdesc
export var
# error -> stop spawning new jobs: --halt soon,fail=1
# "sorted" output: -k
#
parallel -k -j ${ncore} --halt soon,fail=1 processvar ::: `seq ${yrfirst} 1 ${yrlast}</