From d13d34111c56eba77366c76c8561f2abe13e3b58 Mon Sep 17 00:00:00 2001
From: Bianca Wentzel <k202206@l40048.lvt.dkrz.de>
Date: Fri, 17 Jan 2025 10:42:28 +0100
Subject: [PATCH 1/8] First approach to find orog files
---
pet-wrapper-api.py | 53 +++++++++++++++++++++++++++++++++++++++-------
1 file changed, 45 insertions(+), 8 deletions(-)
diff --git a/pet-wrapper-api.py b/pet-wrapper-api.py
index 94c4ef6..5b07f70 100644
--- a/pet-wrapper-api.py
+++ b/pet-wrapper-api.py
@@ -59,13 +59,18 @@ class PET(plugin.PluginAbstract):
######################################################
__parameters__ = ParameterDictionary(
+ SolrField(
+ name="project",
+ facet="project",
+ mandatory=True,
+ ),
SolrField(
name="product",
facet="product",
mandatory=True,
help="Product facet for input data search.",
predefined_facets={
- "project": ["nukleus"],
+ #"project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
}),
@@ -75,7 +80,7 @@ class PET(plugin.PluginAbstract):
mandatory=True,
help="Institute facet for input data search.",
predefined_facets={
- "project": ["nukleus"],
+ #"project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
},
@@ -86,7 +91,7 @@ class PET(plugin.PluginAbstract):
mandatory=True,
help="Model facet for input data search.",
predefined_facets={
- "project": ["nukleus"],
+ #"project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
},
@@ -97,7 +102,7 @@ class PET(plugin.PluginAbstract):
mandatory=True,
help="Experiment facet for input data search.",
predefined_facets={
- "project": ["nukleus"],
+ #"project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
},
@@ -108,7 +113,7 @@ class PET(plugin.PluginAbstract):
mandatory=True,
help="ensemble facet for input data search.",
predefined_facets={
- "project": ["nukleus"],
+ #"project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
},
@@ -139,9 +144,38 @@ class PET(plugin.PluginAbstract):
name="debug",
default=False,
help="Turn on to get better detailed debug information in case you face any issue."
+ ),
+ SolrField(
+ name="alti_file",
+ default=find_orog(config_dict)
)
+
)
+ def find_orog(config_dict):
+ '''
+ Retrieves a list of orographic files based on given facets and returns first entry as suggestion
+
+ Parameters:
+ config_dict (dict): Dictionary containing facet values to search database
+ Returns:
+ orog_files[0] (string): First entry of list of found files
+ '''
+ facets = ["product", "institute", "model", "experiment", "ensemble"]
+ search_args = {"project": config_dict["project"], "variable": "orog"}
+ orog_files = list(freva.databrowser(**search_args))
+
+ for facet in facets:
+ search_args[facet] = config_dict[facet]
+ file_number = freva.count_values(**search_args)
+
+ # if there are results, keep them and the facet
+ if (file_number > 0):
+ orog_files = list(freva.databrowser(**search_args))
+ else:
+ del search_args[facet]
+
+ return orog_files[0]
def find_files(self, config_dict, variables: List[str], time_frequency: str, time: Optional[str] = None):
'''
@@ -157,7 +191,7 @@ class PET(plugin.PluginAbstract):
'''
search_keys = (
- # "project",
+ "project",
"product",
"institute",
"model",
@@ -173,10 +207,13 @@ class PET(plugin.PluginAbstract):
file_dict = {}
for variable in variables:
search_args["variable"] = variable
- if variable == "orog":
- del search_args["ensemble"] ## orog files are part of another ensemble than the other variables
+ #if variable == "orog":
+ # files = find_orog(config_dict)
+ #else:
+ # files = list(freva.databrowser(**search_args))
files = list(freva.databrowser(**search_args))
file_dict[variable] = files
+
return file_dict
--
GitLab
From 91dc342f23118f5a8bce736db403ccfdd546b8c8 Mon Sep 17 00:00:00 2001
From: Bianca Wentzel <k202206@l40048.lvt.dkrz.de>
Date: Fri, 17 Jan 2025 10:49:14 +0100
Subject: [PATCH 2/8] Integrated orog file search into overall database search
---
pet-wrapper-api.py | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
diff --git a/pet-wrapper-api.py b/pet-wrapper-api.py
index 5b07f70..92973b6 100644
--- a/pet-wrapper-api.py
+++ b/pet-wrapper-api.py
@@ -145,11 +145,6 @@ class PET(plugin.PluginAbstract):
default=False,
help="Turn on to get better detailed debug information in case you face any issue."
),
- SolrField(
- name="alti_file",
- default=find_orog(config_dict)
- )
-
)
def find_orog(config_dict):
@@ -207,11 +202,10 @@ class PET(plugin.PluginAbstract):
file_dict = {}
for variable in variables:
search_args["variable"] = variable
- #if variable == "orog":
- # files = find_orog(config_dict)
- #else:
- # files = list(freva.databrowser(**search_args))
- files = list(freva.databrowser(**search_args))
+ if variable == "orog":
+ files = find_orog(config_dict)
+ else:
+ files = list(freva.databrowser(**search_args))
file_dict[variable] = files
--
GitLab
From 1e83e1f4684510afe626f0553dbc00e0c9351514 Mon Sep 17 00:00:00 2001
From: Bianca Wentzel <k202206@l40048.lvt.dkrz.de>
Date: Fri, 17 Jan 2025 10:54:21 +0100
Subject: [PATCH 3/8] Provided missing project definition in tool call
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 540d461..3125bd8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,7 +54,7 @@ func-test:
- export EVALUATION_SYSTEM_PLUGINS=$CI_PROJECT_DIR,pet-wrapper-api
- freva-plugin --list
# TODO: After generalization to all avaiable dataset, we need to provide a func-test here
- - freva-plugin pet product=ceu-3 institute=clmcom-btu model=ec-earth-consortium-ec-earth3-veg-clmcom-btu-icon-2-6-5-rc-nukleus-x2yn2-v1 experiment=historical ensemble=r1i1p1f1 start_date=1967-01-01 end_date=1969-12-31 region=2,6,46,54 output_dir=$CI_PROJECT_DIR/output debug=True -vvv
+ - freva-plugin pet project=nukleus product=ceu-3 institute=clmcom-btu model=ec-earth-consortium-ec-earth3-veg-clmcom-btu-icon-2-6-5-rc-nukleus-x2yn2-v1 experiment=historical ensemble=r1i1p1f1 start_date=1967-01-01 end_date=1969-12-31 region=2,6,46,54 output_dir=$CI_PROJECT_DIR/output debug=True -vvv
- mkdir -p build/func_test
- FREVA_JOB_ID=$(ls plugin_env/freva_output/freva/freva/output/pet/ | head -n 1)
- cd plugin_env/freva_output/freva/freva/output/pet/$FREVA_JOB_ID
@@ -92,7 +92,7 @@ prod-test:
- export EVALUATION_SYSTEM_PLUGINS=$CI_PROJECT_DIR,pet-wrapper-api
- freva-plugin --list
# TODO: After generalization to all avaiable dataset, we need to provide a prod-test here
- - freva-plugin pet product=ceu-3 institute=clmcom-btu model=ec-earth-consortium-ec-earth3-veg-clmcom-btu-icon-2-6-5-rc-nukleus-x2yn2-v1 experiment=historical ensemble=r1i1p1f1 start_date=1970-01-01 end_date=1971-12-31 region=2,6,46,54 caption="PET production test from CI piepline https://gitlab.dkrz.de/ch1187/plugins4freva/pet/-/jobs/$CI_JOB_ID"-vvv
+ - freva-plugin pet project=nukleus product=ceu-3 institute=clmcom-btu model=ec-earth-consortium-ec-earth3-veg-clmcom-btu-icon-2-6-5-rc-nukleus-x2yn2-v1 experiment=historical ensemble=r1i1p1f1 start_date=1970-01-01 end_date=1971-12-31 region=2,6,46,54 caption="PET production test from CI piepline https://gitlab.dkrz.de/ch1187/plugins4freva/pet/-/jobs/$CI_JOB_ID"-vvv
- mkdir -p build/prod_test
- REGIKLIM_JOB_ID=$(ls -t /work/$ACCOUNT/regiklim-work/$GITLAB_USER_LOGIN/regiklim-ces/output/pet/ | head -n 1)
- cd /work/$ACCOUNT/regiklim-work/$GITLAB_USER_LOGIN/regiklim-ces/output/pet/$REGIKLIM_JOB_ID
--
GitLab
From f34de4cf9879f55b7b4d02118d12385d7c07e451 Mon Sep 17 00:00:00 2001
From: Bianca Wentzel <k202206@l40048.lvt.dkrz.de>
Date: Fri, 17 Jan 2025 11:05:23 +0100
Subject: [PATCH 4/8] Fixed function call for org files
---
pet-wrapper-api.py | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/pet-wrapper-api.py b/pet-wrapper-api.py
index 92973b6..868dc02 100644
--- a/pet-wrapper-api.py
+++ b/pet-wrapper-api.py
@@ -147,7 +147,7 @@ class PET(plugin.PluginAbstract):
),
)
- def find_orog(config_dict):
+ def find_orog(self, config_dict):
'''
Retrieves a list of orographic files based on given facets and returns first entry as suggestion
@@ -202,12 +202,8 @@ class PET(plugin.PluginAbstract):
file_dict = {}
for variable in variables:
search_args["variable"] = variable
- if variable == "orog":
- files = find_orog(config_dict)
- else:
- files = list(freva.databrowser(**search_args))
+ files = list(freva.databrowser(**search_args))
file_dict[variable] = files
-
return file_dict
@@ -348,7 +344,7 @@ class PET(plugin.PluginAbstract):
def run_tool(self, config_dict=None):
config_dict["files"] = self.find_files(config_dict, ["tasmax", "tasmin", "sfcWind", "pvap", "rsds"], "day", config_dict["start_date"] + " to " + config_dict["end_date"])
- config_dict["alti_file"] = self.find_files(config_dict, ["orog"], "fx")["orog"]
+ config_dict["alti_file"] = self.find_orog(config_dict)
if "alti_file" not in config_dict or not config_dict["alti_file"]:
logger.error("Can't find altitude file")
--
GitLab
From 3d6b705976a00a6841c7b96e23aae3fe8db661a9 Mon Sep 17 00:00:00 2001
From: Bianca Wentzel <k202206@l40048.lvt.dkrz.de>
Date: Fri, 17 Jan 2025 11:38:03 +0100
Subject: [PATCH 5/8] Removed project field
---
pet-wrapper-api.py | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
diff --git a/pet-wrapper-api.py b/pet-wrapper-api.py
index 868dc02..5c8c652 100644
--- a/pet-wrapper-api.py
+++ b/pet-wrapper-api.py
@@ -59,18 +59,13 @@ class PET(plugin.PluginAbstract):
######################################################
__parameters__ = ParameterDictionary(
- SolrField(
- name="project",
- facet="project",
- mandatory=True,
- ),
SolrField(
name="product",
facet="product",
mandatory=True,
help="Product facet for input data search.",
predefined_facets={
- #"project": ["nukleus"],
+ "project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
}),
@@ -80,7 +75,7 @@ class PET(plugin.PluginAbstract):
mandatory=True,
help="Institute facet for input data search.",
predefined_facets={
- #"project": ["nukleus"],
+ "project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
},
@@ -91,7 +86,7 @@ class PET(plugin.PluginAbstract):
mandatory=True,
help="Model facet for input data search.",
predefined_facets={
- #"project": ["nukleus"],
+ "project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
},
@@ -102,7 +97,7 @@ class PET(plugin.PluginAbstract):
mandatory=True,
help="Experiment facet for input data search.",
predefined_facets={
- #"project": ["nukleus"],
+ "project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
},
@@ -113,7 +108,7 @@ class PET(plugin.PluginAbstract):
mandatory=True,
help="ensemble facet for input data search.",
predefined_facets={
- #"project": ["nukleus"],
+ "project": ["nukleus"],
"variable": ["rsds"],
"time_frequency": ["1day"],
},
--
GitLab
From dd1628f09a4ccbe4c312e36bf34e52514574c0fe Mon Sep 17 00:00:00 2001
From: Bianca Wentzel <k202206@l40048.lvt.dkrz.de>
Date: Fri, 17 Jan 2025 11:42:38 +0100
Subject: [PATCH 6/8] Removed project facet
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3125bd8..540d461 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,7 +54,7 @@ func-test:
- export EVALUATION_SYSTEM_PLUGINS=$CI_PROJECT_DIR,pet-wrapper-api
- freva-plugin --list
# TODO: After generalization to all avaiable dataset, we need to provide a func-test here
- - freva-plugin pet project=nukleus product=ceu-3 institute=clmcom-btu model=ec-earth-consortium-ec-earth3-veg-clmcom-btu-icon-2-6-5-rc-nukleus-x2yn2-v1 experiment=historical ensemble=r1i1p1f1 start_date=1967-01-01 end_date=1969-12-31 region=2,6,46,54 output_dir=$CI_PROJECT_DIR/output debug=True -vvv
+ - freva-plugin pet product=ceu-3 institute=clmcom-btu model=ec-earth-consortium-ec-earth3-veg-clmcom-btu-icon-2-6-5-rc-nukleus-x2yn2-v1 experiment=historical ensemble=r1i1p1f1 start_date=1967-01-01 end_date=1969-12-31 region=2,6,46,54 output_dir=$CI_PROJECT_DIR/output debug=True -vvv
- mkdir -p build/func_test
- FREVA_JOB_ID=$(ls plugin_env/freva_output/freva/freva/output/pet/ | head -n 1)
- cd plugin_env/freva_output/freva/freva/output/pet/$FREVA_JOB_ID
@@ -92,7 +92,7 @@ prod-test:
- export EVALUATION_SYSTEM_PLUGINS=$CI_PROJECT_DIR,pet-wrapper-api
- freva-plugin --list
# TODO: After generalization to all avaiable dataset, we need to provide a prod-test here
- - freva-plugin pet project=nukleus product=ceu-3 institute=clmcom-btu model=ec-earth-consortium-ec-earth3-veg-clmcom-btu-icon-2-6-5-rc-nukleus-x2yn2-v1 experiment=historical ensemble=r1i1p1f1 start_date=1970-01-01 end_date=1971-12-31 region=2,6,46,54 caption="PET production test from CI piepline https://gitlab.dkrz.de/ch1187/plugins4freva/pet/-/jobs/$CI_JOB_ID"-vvv
+ - freva-plugin pet product=ceu-3 institute=clmcom-btu model=ec-earth-consortium-ec-earth3-veg-clmcom-btu-icon-2-6-5-rc-nukleus-x2yn2-v1 experiment=historical ensemble=r1i1p1f1 start_date=1970-01-01 end_date=1971-12-31 region=2,6,46,54 caption="PET production test from CI piepline https://gitlab.dkrz.de/ch1187/plugins4freva/pet/-/jobs/$CI_JOB_ID"-vvv
- mkdir -p build/prod_test
- REGIKLIM_JOB_ID=$(ls -t /work/$ACCOUNT/regiklim-work/$GITLAB_USER_LOGIN/regiklim-ces/output/pet/ | head -n 1)
- cd /work/$ACCOUNT/regiklim-work/$GITLAB_USER_LOGIN/regiklim-ces/output/pet/$REGIKLIM_JOB_ID
--
GitLab
From 6170b1c710f8b1645ca0e83897aa3070d77ae237 Mon Sep 17 00:00:00 2001
From: Bianca Wentzel <k202206@l40048.lvt.dkrz.de>
Date: Fri, 17 Jan 2025 11:49:38 +0100
Subject: [PATCH 7/8] Set project for orog serach
---
pet-wrapper-api.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pet-wrapper-api.py b/pet-wrapper-api.py
index 5c8c652..6877c01 100644
--- a/pet-wrapper-api.py
+++ b/pet-wrapper-api.py
@@ -152,7 +152,8 @@ class PET(plugin.PluginAbstract):
orog_files[0] (string): First entry of list of found files
'''
facets = ["product", "institute", "model", "experiment", "ensemble"]
- search_args = {"project": config_dict["project"], "variable": "orog"}
+ #search_args = {"project": config_dict["project"], "variable": "orog"} for later if project facte is given
+ search_args = {"project": "nukleus", "variable": "orog"}
orog_files = list(freva.databrowser(**search_args))
for facet in facets:
--
GitLab
From 9a1be8224364f035f25a080f90849d4a7a7296d5 Mon Sep 17 00:00:00 2001
From: Bianca Wentzel <k202206@l40048.lvt.dkrz.de>
Date: Fri, 17 Jan 2025 11:59:00 +0100
Subject: [PATCH 8/8] Removed project facet from database search
---
pet-wrapper-api.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pet-wrapper-api.py b/pet-wrapper-api.py
index 6877c01..02a390b 100644
--- a/pet-wrapper-api.py
+++ b/pet-wrapper-api.py
@@ -182,7 +182,7 @@ class PET(plugin.PluginAbstract):
'''
search_keys = (
- "project",
+ #"project",
"product",
"institute",
"model",
--
GitLab