Updated readme

README.md

@@ -4,35 +4,24 @@ This repo contains material for **preparing, scheduling and archiving** intake-e
 
 Cronjobs running for generating intake functionalities include
 
-- updating catalogs for the cmip data pool of dkrz
-- testing catalogs
-- hosting catalogs at /pool/data/catalogs and in the cloud
-- archiving catalogs
+- updating catalogs for the cmip data pool of dkrz (`builder/dkrz_PROJECT_STORE.py`)
+- testing catalogs (`test/check_load_catalog_PROJECT.py`)
+- hosting and archiving catalogs at /pool/data/catalogs and in the cloud (`archive-catalog.sh`)
 - creating statistics for catalogs including kpis like no. of files and datasets
 
 One **main catalog** collects all catalogs in /pool/data/catalogs and serves as the *entry point* for dkrz's intake users.
 
-## builder/
+## environment.yml
 
-This folder contains scripts for generating the catalog data bases (`.csv.gz`).
+Use that file with `conda env create -f environment.yml` to generate a software environment which allows you to use the notebooks wihtin this repository.
 
 ## esm-collections/
 
 All **esm-collections** available at DKRZ are saved within this folder. Those are `.json` files which can be opend with `intake.open_esm_datastore()`.
 
-## environment.yml
-
-Use that file with `conda env create -f environment.yml` to generate a software environment which allows you to use the notebooks wihtin this repository.
-
-## archive-catalog.sh
-
-This script is part of the updating cronjob. It
+## builder/
 
-* test the newly created catalogs
-* writes those catalogs to
-    * a place which is linked to /pool/data/catalogs
-    * to the swift cloud store
-* archives the old version of the catalog
+This folder contains scripts for generating the catalog data bases (`.csv.gz`).
 
 ## tests/
 

archive-catalog.sh

@@ -9,26 +9,17 @@ oldcatalogArchDir="${path}/Catalogs/archive"
 mkdir -p ${oldcatalogArchDir}
 oldcatalogArchzip=${path}/Catalogs/archive/dkrz_${project}_disk_$(date +"%Y-%m-%d" -d "yesterday").csv.gz
 #
-python /home/k/k204210/intake-esm/tests/check_load_catalog_${project}.py
-if [ $? == 0 ]; then
-  chmod 774 ${newcatalogzip}
-  cp ${oldcatalogzip} ${oldcatalogArchzip}
-  cp $newcatalogzip $oldcatalogzip
-  if [ $? != 0 ]; then
-    cp ${oldcatalogArchzip} ${oldcatalogzip}
-  else
-    datemon=$(date +"%m")
-    mon=$((datemon-1))
-    if [ $mon == 0 ]; then
-      mon=12
-    fi
-    printf -v mon "%02d" $mon
-    rm -f $(ls -d ${oldcatalogArchDir}/* | grep -v "\-${mon}\-") #-f for if no file exists, -d for entire path
-  fi
-  python /home/k/k204210/intake-esm/catalog-to-swift.py ${newcatalogzip}
-  if [ $? != 0 ]; then
-    echo "Could not upload to swift"
-  fi
+chmod 774 ${newcatalogzip}
+cp ${oldcatalogzip} ${oldcatalogArchzip}
+cp $newcatalogzip $oldcatalogzip
+if [ $? != 0 ]; then
+  cp ${oldcatalogArchzip} ${oldcatalogzip}
 else
-  echo "Tests for new catalog file failed." &&  exit 1
+  datemon=$(date +"%m")
+  mon=$((datemon-1))
+  if [ $mon == 0 ]; then
+    mon=12
+  fi
+  printf -v mon "%02d" $mon
+  rm -f $(ls -d ${oldcatalogArchDir}/* | grep -v "\-${mon}\-") #-f for if no file exists, -d for entire path
 fi