Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
I
intake-esm-datastore
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Aaron Spring
intake-esm-datastore
Commits
9a577c56
Commit
9a577c56
authored
Oct 12, 2019
by
Anderson Banihirwe
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
update catalogs and add json files
parent
3c237372
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
421 additions
and
32 deletions
+421
-32
builders/cmip6_catalog_builder.ipynb
builders/cmip6_catalog_builder.ipynb
+273
-32
catalogs/glade-cmip6-dcpp.csv.gz
catalogs/glade-cmip6-dcpp.csv.gz
+0
-0
catalogs/glade-cmip6-dcpp.json
catalogs/glade-cmip6-dcpp.json
+79
-0
catalogs/glade-cmip6.csv.gz
catalogs/glade-cmip6.csv.gz
+0
-0
catalogs/glade-cmip6.json
catalogs/glade-cmip6.json
+69
-0
No files found.
builders/cmip6_catalog_builder.ipynb
View file @
9a577c56
...
...
@@ -91,6 +91,37 @@
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['CFMIP',\n",
" 'CMIP',\n",
" 'LUMIP',\n",
" 'LS3MIP',\n",
" 'OMIP',\n",
" 'HighResMIP',\n",
" 'DCPP',\n",
" 'AerChemMIP',\n",
" 'PAMIP',\n",
" 'ScenarioMIP']"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"activity_ids = list(Path(persist_path).rglob(\"*.txt\"))\n",
"activity_ids = [activity_id.stem for activity_id in activity_ids]\n",
"activity_ids"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
...
...
@@ -150,7 +181,7 @@
"4 /glade/collections/cmip/CMIP6/AerChemMIP/BCC/B..."
]
},
"execution_count":
4
,
"execution_count":
5
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -163,7 +194,7 @@
},
{
"cell_type": "code",
"execution_count":
5
,
"execution_count":
6
,
"metadata": {},
"outputs": [
{
...
...
@@ -172,7 +203,7 @@
"1027617"
]
},
"execution_count":
5
,
"execution_count":
6
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -183,7 +214,7 @@
},
{
"cell_type": "code",
"execution_count":
6
,
"execution_count":
7
,
"metadata": {},
"outputs": [],
"source": [
...
...
@@ -231,15 +262,15 @@
},
{
"cell_type": "code",
"execution_count":
7
,
"execution_count":
8
,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 3.
31 s, sys: 0 ns, total: 3.31
s\n",
"Wall time: 3.
31
s\n"
"CPU times: user 3.
66 s, sys: 13.2 ms, total: 3.67
s\n",
"Wall time: 3.
68
s\n"
]
}
],
...
...
@@ -251,7 +282,7 @@
},
{
"cell_type": "code",
"execution_count":
8
,
"execution_count":
9
,
"metadata": {},
"outputs": [
{
...
...
@@ -260,7 +291,7 @@
"609911"
]
},
"execution_count":
8
,
"execution_count":
9
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -271,7 +302,7 @@
},
{
"cell_type": "code",
"execution_count":
9
,
"execution_count":
10
,
"metadata": {},
"outputs": [],
"source": [
...
...
@@ -306,15 +337,15 @@
},
{
"cell_type": "code",
"execution_count": 1
0
,
"execution_count": 1
1
,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 1
3.5 s, sys: 560 ms, total: 14
.1 s\n",
"Wall time: 1
4
.1 s\n"
"CPU times: user 1
5.4 s, sys: 622 ms, total: 16
.1 s\n",
"Wall time: 1
6
.1 s\n"
]
}
],
...
...
@@ -325,7 +356,7 @@
},
{
"cell_type": "code",
"execution_count": 1
1
,
"execution_count": 1
2
,
"metadata": {},
"outputs": [
{
...
...
@@ -344,7 +375,7 @@
" 'path': '/glade/collections/cmip/CMIP6/AerChemMIP/BCC/BCC-ESM1/ssp370/r2i1p1f1/Amon/hfls/gn/v20190624/hfls/hfls_Amon_BCC-ESM1_ssp370_r2i1p1f1_gn_201501-205512.nc'}"
]
},
"execution_count": 1
1
,
"execution_count": 1
2
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -355,7 +386,7 @@
},
{
"cell_type": "code",
"execution_count": 1
2
,
"execution_count": 1
3
,
"metadata": {},
"outputs": [
{
...
...
@@ -364,7 +395,7 @@
"609911"
]
},
"execution_count": 1
2
,
"execution_count": 1
3
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -375,7 +406,7 @@
},
{
"cell_type": "code",
"execution_count": 1
3
,
"execution_count": 1
4
,
"metadata": {},
"outputs": [
{
...
...
@@ -510,7 +541,7 @@
"4 /glade/collections/cmip/CMIP6/AerChemMIP/BCC/B... "
]
},
"execution_count": 1
3
,
"execution_count": 1
4
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -523,7 +554,7 @@
},
{
"cell_type": "code",
"execution_count": 1
4
,
"execution_count": 1
5
,
"metadata": {},
"outputs": [
{
...
...
@@ -532,7 +563,217 @@
"609911"
]
},
"execution_count": 14,
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(df)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"invalids = df[~df.activity_id.isin(activity_ids)]\n",
"df = df[df.activity_id.isin(activity_ids)]"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>variable_id</th>\n",
" <th>table_id</th>\n",
" <th>source_id</th>\n",
" <th>experiment_id</th>\n",
" <th>member_id</th>\n",
" <th>grid_label</th>\n",
" <th>time_range</th>\n",
" <th>activity_id</th>\n",
" <th>institution_id</th>\n",
" <th>version</th>\n",
" <th>path</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>507544</td>\n",
" <td>sftof</td>\n",
" <td>Ofx</td>\n",
" <td>historical</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>gn</td>\n",
" <td>NaN</td>\n",
" <td>NCC</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>v20190815</td>\n",
" <td>/glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2...</td>\n",
" </tr>\n",
" <tr>\n",
" <td>507546</td>\n",
" <td>basin</td>\n",
" <td>Ofx</td>\n",
" <td>historical</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>gn</td>\n",
" <td>NaN</td>\n",
" <td>NCC</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>v20190815</td>\n",
" <td>/glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2...</td>\n",
" </tr>\n",
" <tr>\n",
" <td>507549</td>\n",
" <td>volcello</td>\n",
" <td>Ofx</td>\n",
" <td>historical</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>gr</td>\n",
" <td>NaN</td>\n",
" <td>NCC</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>v20190815</td>\n",
" <td>/glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2...</td>\n",
" </tr>\n",
" <tr>\n",
" <td>508393</td>\n",
" <td>areacello</td>\n",
" <td>Ofx</td>\n",
" <td>piControl</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>gn</td>\n",
" <td>NaN</td>\n",
" <td>NCC</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>v20190815</td>\n",
" <td>/glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2...</td>\n",
" </tr>\n",
" <tr>\n",
" <td>508395</td>\n",
" <td>basin</td>\n",
" <td>Ofx</td>\n",
" <td>piControl</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>gn</td>\n",
" <td>NaN</td>\n",
" <td>NCC</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>v20190815</td>\n",
" <td>/glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2...</td>\n",
" </tr>\n",
" <tr>\n",
" <td>508396</td>\n",
" <td>volcello</td>\n",
" <td>Ofx</td>\n",
" <td>piControl</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>r1i1p1f1</td>\n",
" <td>gr</td>\n",
" <td>NaN</td>\n",
" <td>NCC</td>\n",
" <td>NorESM2-LM</td>\n",
" <td>v20190815</td>\n",
" <td>/glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2...</td>\n",
" </tr>\n",
" <tr>\n",
" <td>509418</td>\n",
" <td>thetao</td>\n",
" <td>Omon</td>\n",
" <td>PCMDI-test-1-0</td>\n",
" <td>piControl-withism</td>\n",
" <td>r3i1p1f1</td>\n",
" <td>gn</td>\n",
" <td>016201-016201</td>\n",
" <td>v20190926</td>\n",
" <td>thetao</td>\n",
" <td>v20190926</td>\n",
" <td>/glade/collections/cmip/CMIP6/CMIP/FIO-QLNM/FI...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" variable_id table_id source_id experiment_id member_id \\\n",
"507544 sftof Ofx historical NorESM2-LM r1i1p1f1 \n",
"507546 basin Ofx historical NorESM2-LM r1i1p1f1 \n",
"507549 volcello Ofx historical NorESM2-LM r1i1p1f1 \n",
"508393 areacello Ofx piControl NorESM2-LM r1i1p1f1 \n",
"508395 basin Ofx piControl NorESM2-LM r1i1p1f1 \n",
"508396 volcello Ofx piControl NorESM2-LM r1i1p1f1 \n",
"509418 thetao Omon PCMDI-test-1-0 piControl-withism r3i1p1f1 \n",
"\n",
" grid_label time_range activity_id institution_id version \\\n",
"507544 gn NaN NCC NorESM2-LM v20190815 \n",
"507546 gn NaN NCC NorESM2-LM v20190815 \n",
"507549 gr NaN NCC NorESM2-LM v20190815 \n",
"508393 gn NaN NCC NorESM2-LM v20190815 \n",
"508395 gn NaN NCC NorESM2-LM v20190815 \n",
"508396 gr NaN NCC NorESM2-LM v20190815 \n",
"509418 gn 016201-016201 v20190926 thetao v20190926 \n",
"\n",
" path \n",
"507544 /glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2... \n",
"507546 /glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2... \n",
"507549 /glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2... \n",
"508393 /glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2... \n",
"508395 /glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2... \n",
"508396 /glade/collections/cmip/CMIP6/CMIP/NCC/NorESM2... \n",
"509418 /glade/collections/cmip/CMIP6/CMIP/FIO-QLNM/FI... "
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"invalids"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"609904"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -552,7 +793,7 @@
},
{
"cell_type": "code",
"execution_count": 1
5
,
"execution_count": 1
9
,
"metadata": {},
"outputs": [],
"source": [
...
...
@@ -562,7 +803,7 @@
},
{
"cell_type": "code",
"execution_count":
16
,
"execution_count":
20
,
"metadata": {},
"outputs": [
{
...
...
@@ -697,7 +938,7 @@
"4 /glade/collections/cmip/CMIP6/AerChemMIP/BCC/B... "
]
},
"execution_count":
16
,
"execution_count":
20
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -711,7 +952,7 @@
},
{
"cell_type": "code",
"execution_count":
17
,
"execution_count":
21
,
"metadata": {},
"outputs": [],
"source": [
...
...
@@ -721,7 +962,7 @@
},
{
"cell_type": "code",
"execution_count":
18
,
"execution_count":
22
,
"metadata": {},
"outputs": [
{
...
...
@@ -856,7 +1097,7 @@
"4 /glade/collections/cmip/CMIP6/DCPP/CCCma/CanES... "
]
},
"execution_count":
18
,
"execution_count":
22
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -867,7 +1108,7 @@
},
{
"cell_type": "code",
"execution_count":
19
,
"execution_count":
23
,
"metadata": {},
"outputs": [
{
...
...
@@ -1008,7 +1249,7 @@
"4 /glade/collections/cmip/CMIP6/DCPP/CCCma/CanES... 1975 "
]
},
"execution_count":
19
,
"execution_count":
23
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -1021,7 +1262,7 @@
},
{
"cell_type": "code",
"execution_count": 2
0
,
"execution_count": 2
4
,
"metadata": {},
"outputs": [
{
...
...
@@ -1162,7 +1403,7 @@
"4 /glade/collections/cmip/CMIP6/DCPP/CCCma/CanES... "
]
},
"execution_count": 2
0
,
"execution_count": 2
4
,
"metadata": {},
"output_type": "execute_result"
}
...
...
@@ -1176,7 +1417,7 @@
},
{
"cell_type": "code",
"execution_count": 2
1
,
"execution_count": 2
5
,
"metadata": {},
"outputs": [],
"source": [
...
...
catalogs/glade-cmip6-dcpp.csv.gz
View file @
9a577c56
No preview for this file type
catalogs/glade-cmip6-dcpp.json
0 → 100644
View file @
9a577c56
{
"esmcat_version"
:
"0.1.0"
,
"id"
:
"glade-cmip6-dcpp"
,
"description"
:
"This is an ESM collection for CMIP6 Decadal Climate Prediction data accessible on the NCAR's GLADE disk storage system in /glade/collections/cmip/CMIP6"
,
"catalog_file"
:
"/glade/collections/cmip/catalog/intake-esm-datastore/catalogs/glade-cmip6-dcpp.csv.gz"
,
"attributes"
:
[
{
"column_name"
:
"activity_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_activity_id.json"
},
{
"column_name"
:
"source_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_source_id.json"
},
{
"column_name"
:
"institution_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_institution_id.json"
},
{
"column_name"
:
"experiment_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_experiment_id.json"
},
{
"column_name"
:
"member_id"
,
"vocabulary"
:
""
},
{
"column_name"
:
"table_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_table_id.json"
},
{
"column_name"
:
"variable_id"
,
"vocabulary"
:
""
},
{
"column_name"
:
"grid_label"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_grid_label.json"
},
{
"column_name"
:
"start_year"
,
"vocabulary"
:
""
},
{
"column_name"
:
"version"
,
"vocabulary"
:
""
}
],
"assets"
:
{
"column_name"
:
"path"
,
"format"
:
"netcdf"
},
"aggregation_control"
:
{
"variable_column_name"
:
"variable_id"
,
"groupby_attrs"
:
[
"activity_id"
,
"institution_id"
,
"source_id"
,
"experiment_id"
,
"table_id"
,
"grid_label"
],
"aggregations"
:
[
{
"type"
:
"join_new"
,
"attribute_name"
:
"member_id"
,
"options"
:
{
"coords"
:
"minimal"
,
"compat"
:
"override"
}
},
{
"type"
:
"join_new"
,
"attribute_name"
:
"start_year"
,
"options"
:
{
"coords"
:
"minimal"
,
"compat"
:
"override"
}
},
{
"type"
:
"join_existing"
,
"attribute_name"
:
"time_range"
,
"options"
:
{
"dim"
:
"time"
}
},
{
"type"
:
"union"
,
"attribute_name"
:
"variable_id"
}
]
}
}
catalogs/glade-cmip6.csv.gz
View file @
9a577c56
No preview for this file type
catalogs/glade-cmip6.json
0 → 100644
View file @
9a577c56
{
"esmcat_version"
:
"0.1.0"
,
"id"
:
"glade-cmip6"
,
"description"
:
"This is an ESM collection for CMIP6 data accessible on the NCAR's GLADE disk storage system in /glade/collections/cmip/CMIP6"
,
"catalog_file"
:
"/glade/collections/cmip/catalog/intake-esm-datastore/catalogs/glade-cmip6.csv.gz"
,
"attributes"
:
[
{
"column_name"
:
"activity_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_activity_id.json"
},
{
"column_name"
:
"source_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_source_id.json"
},
{
"column_name"
:
"institution_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_institution_id.json"
},
{
"column_name"
:
"experiment_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_experiment_id.json"
},
{
"column_name"
:
"member_id"
,
"vocabulary"
:
""
},
{
"column_name"
:
"table_id"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_table_id.json"
},
{
"column_name"
:
"variable_id"
,
"vocabulary"
:
""
},
{
"column_name"
:
"grid_label"
,
"vocabulary"
:
"https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_grid_label.json"
},
{
"column_name"
:
"version"
,
"vocabulary"
:
""
}
],
"assets"
:
{
"column_name"
:
"path"
,
"format"
:
"netcdf"
},
"aggregation_control"
:
{
"variable_column_name"
:
"variable_id"
,
"groupby_attrs"
:
[
"activity_id"
,
"institution_id"
,
"source_id"
,
"experiment_id"
,
"table_id"
,
"grid_label"
],
"aggregations"
:
[
{
"type"
:
"join_new"
,
"attribute_name"
:
"member_id"
,
"options"
:
{
"coords"
:
"minimal"
,
"compat"
:
"override"
}
},
{
"type"
:
"join_existing"
,
"attribute_name"
:
"time_range"
,
"options"
:
{
"dim"
:
"time"
}
},
{
"type"
:
"union"
,
"attribute_name"
:
"variable_id"
}
]
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment