diff --git a/notebooks/intake-esm_multi-var.ipynb b/notebooks/intake-esm_multi-var.ipynb index 953e3dd6407fe2abf5c922250d3dafac8d55495c..6c0872ed574afa230e823ab9ae828bda9e01f5e5 100644 --- a/notebooks/intake-esm_multi-var.ipynb +++ b/notebooks/intake-esm_multi-var.ipynb @@ -14,21 +14,30 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# basepath for all your experiments\n", - "exp_path = '/work/bm1124/m300524/experiments'" + "import os\n", + "import subprocess\n", + "from tqdm.auto import tqdm\n", + "from pathlib import Path\n", + "import shutil\n", + "import dask.dataframe as dd\n", + "import fnmatch\n", + "import pandas as pd\n", + "import cdo\n", + "cdo=cdo.Cdo()" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "persist_path='filenames'" + "# basepath for all your experiments\n", + "exp_path = '/work/bm1124/m300524/experiments'" ] }, { @@ -37,7 +46,8 @@ "metadata": {}, "outputs": [], "source": [ - "import shutil\n", + "persist_path='filenames'\n", + "\n", "shutil.rmtree(persist_path)" ] }, @@ -47,11 +57,6 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import re\n", - "import subprocess\n", - "from tqdm.auto import tqdm\n", - "from pathlib import Path\n", "def get_file_list(exppath,expids,persist_path):\n", " persist_path = Path(persist_path)\n", " persist_path.mkdir(exist_ok=True)\n", @@ -191,7 +196,6 @@ } ], "source": [ - "import dask.dataframe as dd\n", "df = dd.read_csv(f\"{persist_path}/*.txt\", header=None).compute()\n", "df.columns = [\"path\"]\n", "df.head()" @@ -215,7 +219,7 @@ "%%time\n", "# exclude paths, here cmorized variables by c6_cmor_land.sh\n", "exclude_patterns = ['*Lmon*']\n", - "import fnmatch\n", + "\n", "def _filter_func(path):\n", " return not any(\n", " fnmatch.fnmatch(path, pat=exclude_pattern) for exclude_pattern in exclude_patterns\n", @@ -231,8 +235,6 @@ "metadata": {}, "outputs": [], "source": [ - "from intake.source.utils import reverse_format\n", - " \n", "def get_attrs(filepath):\n", " \"\"\"Get attributes based on filepath.\"\"\"\n", " basename = os.path.basename(filepath)\n", @@ -422,7 +424,6 @@ } ], "source": [ - "import pandas as pd\n", "df1 = pd.DataFrame(entries)\n", "\n", "df1.head()" @@ -477,8 +478,7 @@ ], "source": [ "# add variable ids\n", - "import cdo\n", - "cdo=cdo.Cdo()\n", + "\n", "def add_variable_ids(entry):\n", " model = entry['model']\n", " stream = entry['stream']\n", @@ -683,6 +683,7 @@ } ], "source": [ + "# number of files in catalog\n", "df1.index.size" ] },