diff --git a/notebooks/intake_esm_CMIP6_mistral_stats.ipynb b/notebooks/intake_esm_CMIP6_mistral_stats.ipynb index 0a39441d884fe1aa2c54319b9674dc2f18d2f32f..1993c56e15faba459f1302c85ade35e63a1f179a 100644 --- a/notebooks/intake_esm_CMIP6_mistral_stats.ipynb +++ b/notebooks/intake_esm_CMIP6_mistral_stats.ipynb @@ -1,86 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext lab_black" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of CPUs: 72, number of threads: 6, number of workers: 12\n" - ] - }, - { - "data": { - "text/html": [ - "<table style=\"border: 2px solid white;\">\n", - "<tr>\n", - "<td style=\"vertical-align: top; border: 0px solid white\">\n", - "<h3 style=\"text-align: left;\">Client</h3>\n", - "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n", - " <li><b>Scheduler: </b>inproc://10.50.44.48/31834/41</li>\n", - " <li><b>Dashboard: </b><a href='http://localhost:8888/proxy/40729/status' target='_blank'>http://localhost:8888/proxy/40729/status</a></li>\n", - "</ul>\n", - "</td>\n", - "<td style=\"vertical-align: top; border: 0px solid white\">\n", - "<h3 style=\"text-align: left;\">Cluster</h3>\n", - "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n", - " <li><b>Workers: </b>12</li>\n", - " <li><b>Cores: </b>72</li>\n", - " <li><b>Memory: </b>797.25 GB</li>\n", - "</ul>\n", - "</td>\n", - "</tr>\n", - "</table>" - ], - "text/plain": [ - "<Client: 'inproc://10.50.44.48/31834/41' processes=12 threads=72, memory=797.25 GB>" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from dask.distributed import Client\n", - "import multiprocessing\n", - "\n", - "ncpu = multiprocessing.cpu_count()\n", - "threads = 6\n", - "nworker = ncpu // threads\n", - "print(\n", - " f\"Number of CPUs: {ncpu}, number of threads: {threads}, number of workers: {nworker}\"\n", - ")\n", - "\n", - "client = Client(\n", - " processes=False, threads_per_worker=threads, n_workers=nworker, memory_limit=\"256GB\"\n", - ")\n", - "client" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Intake to load CMIP data\n", - "\n", - "### Using intake-esm on mistral\n", - "- install intake-esm: https://intake-esm.readthedocs.io/en/latest/installation.html\n", - "- check the already built catalogs: `/work/ik1017/Catalogs/` or `https://github.com/NCAR/intake-esm-datastore/` and skip long catalog building process of running the builder" - ] - }, { "cell_type": "code", "execution_count": 25, @@ -106,24 +25,6 @@ "# CMIP6" ] }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "#!ls -all /work/ik1017/Catalogs/mistral-cmip6.*" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [], - "source": [ - "#!cat /work/ik1017/Catalogs/mistral-cmip6.json" - ] - }, { "cell_type": "code", "execution_count": 164, @@ -134,15 +35,6 @@ "plt.rcParams['figure.figsize'] = 10, 4" ] }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, { "cell_type": "code", "execution_count": 27,