Commit 60428eb6 authored by Mahesh Ramadoss's avatar Mahesh Ramadoss
Browse files

Upload plugin analytics code

parent 37f42952
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Plugin Data Analytics\n",
"\n",
"This notebook demonstrate the code that analyse the utilization of plugins and its atrributes in the climextreme project."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##### import Packages\n",
"~~~ python\n",
"import mysql.connector\n",
"import numpy as np\n",
"import json\n",
"from collections import Counter\n",
"~~~"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Call mysql connecter method with DB connection parameters\n",
"~~~python\n",
"mysqb=mysql.connector.connect(host=\"<hostname or ip>\",user=\"xxxxxxx\",password=\"xxxxxx\",database=\"<db name>\")\n",
"record=mysqb.cursor()\n",
"~~~"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### ### Construct the query \n",
"##### Example query: Retrieve the number of records which have used CMIP5 model data in timestamp between 2020 and 2021\n",
"\n",
"~~~python\n",
"#####Construct Query with search constraints\n",
"query=\"select configuration from history_history where configuration like '%\\\"project\\\": \\\"cmip5\\\"%' and timestamp between '2020-01-01' and '2021-04-19'\"\n",
"####Query Execution #####\n",
"~~~"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Fetch the records and assign into numpy array\n",
"~~~python\n",
"record.execute(query)\n",
"result=record.fetchall()\n",
"config=np.array(result)\n",
"~~~"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Parsing the configuration details using Json package\n",
"\n",
"~~~python\n",
"#### numpy array variable declaration######\n",
"experiment=np.array([])\n",
"variables=np.array([])\n",
"time_freq=np.array([])\n",
"\n",
"### Retrieve the record######\n",
"for con in config:\n",
"\t\n",
"\tdicts=json.loads(con[0])\n",
"\t#### Parsing the experiment attribute #######\n",
"\tif \"experiment\" in dicts:\n",
"\t\texp=dicts.get(\"experiment\")\n",
"\t\tif exp is not None:\n",
"\t\t\t#### Store the Experiment name in the array experiment\n",
"\t\t\texperiment=np.append(experiment,exp)\n",
"\t#### Parsing the variable attribute #######\n",
"\tif \"variable\" in dicts:\n",
"\t\tvar=dicts.get(\"variable\")\n",
"\t\tif var is not None:\n",
"\t\tvariables=np.append(variables,var)\n",
"\t#### Parsing the time_frequency attribute #######\n",
"\tif \"time_frequency\" in dicts:\n",
"\t\ttf=dicts.get(\"time_frequency\")\n",
"\t\tif tf is not None:\n",
" \ttime_freq=np.append(time_freq,tf)\n",
"~~~"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Perform the statistics report generation of attribute usage\n",
"~~~ python\n",
"##### Print the attributes and its count #######\n",
"print(\"Experiment\\n\")\n",
"for key,value in dict(Counter(experiment)).items():\n",
"\tprint(key+\"(\"+str(value)+\")\")\n",
"print(\"\\nVariable\\n\")\n",
"for key, value in dict(Counter(variables)).items():\n",
"\tprint(key+\"(\"+str(value)+\")\")\n",
"print(\"\\nTime Frequency\\n\")\n",
"for key, value in dict(Counter(time_freq)).items():\n",
"\tprint(key+\"(\"+str(value)+\")\")\n",
"~~~\n",
"\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Retrieve the list of plugin with number of times of usage in timestamp between 2020 and 2021\n",
"~~~python\n",
"#####Plugin######\n",
"print(\"\\nPlugins\\n\")\n",
"#####Query for plugin name retrival#########\n",
"query=\"select tool from history_history where configuration like '%\\\"project\\\": \\\"cmip5\\\"%' and timestamp between '2020-01-01' and '2021-04-19'\"\n",
"record.execute(query)\n",
"result=record.fetchall()\n",
"plugin=np.array(result)\n",
"##### Print plugin names and its count####\n",
"for key,value in dict(Counter(plugin.ravel())).items():\n",
" print(key+\"(\"+str(value)+\")\")\n",
"~~~"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Complete code"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"~~~python\n",
"import mysql.connector\n",
"import numpy as np\n",
"import json\n",
"from collections import Counter\n",
"\n",
"#####Database Connection###########\n",
"mysqb=mysql.connector.connect(host=\"<hostname or ip>\",user=\"xxxxxxx\",password=\"xxxxxx\",database=\"<db name>\")\n",
"record=mysqb.cursor()\n",
"#####Construct Query with search constraints\n",
"query=\"select configuration from history_history where configuration like '%\\\"project\\\": \\\"cmip5\\\"%' and timestamp between '2020-01-01' and '2021-04-19'\"\n",
"####Query Execution #####\n",
"record.execute(query)\n",
"result=record.fetchall()\n",
"config=np.array(result)\n",
"#### numpy array variable declaration######\n",
"experiment=np.array([])\n",
"variables=np.array([])\n",
"time_freq=np.array([])\n",
"\n",
"### Retrieve the record######\n",
"for con in config:\n",
"\t#print(con[0])\n",
"\tdicts=json.loads(con[0])\n",
"\t#### Parsing the experiment attribute #######\n",
"\tif \"experiment\" in dicts:\n",
"\t\texp=dicts.get(\"experiment\")\n",
"\t\tif exp is not None:\n",
"\t\t\t#### Store the Experiment name in the array experiment\n",
"\t\t\texperiment=np.append(experiment,exp)\n",
"\t#### Parsing the variable attribute #######\n",
"\tif \"variable\" in dicts:\n",
"\t\tvar=dicts.get(\"variable\")\n",
"\t\tif var is not None:\n",
"\t\tvariables=np.append(variables,var)\n",
"\t#### Parsing the time_frequency attribute #######\n",
"\tif \"time_frequency\" in dicts:\n",
"\t\ttf=dicts.get(\"time_frequency\")\n",
"\t\tif tf is not None:\n",
" \ttime_freq=np.append(time_freq,tf)\n",
"##### Print the attributes and its count #######\n",
"print(\"Experiment\\n\")\n",
"for key,value in dict(Counter(experiment)).items():\n",
"\tprint(key+\"(\"+str(value)+\")\")\n",
"print(\"\\nVariable\\n\")\n",
"for key, value in dict(Counter(variables)).items():\n",
"\tprint(key+\"(\"+str(value)+\")\")\n",
"print(\"\\nTime Frequency\\n\")\n",
"for key, value in dict(Counter(time_freq)).items():\n",
"\tprint(key+\"(\"+str(value)+\")\")\n",
"#####Plugin######\n",
"print(\"\\nPlugins\\n\")\n",
"#####Query for plugin name retrival#########\n",
"query=\"select tool from history_history where configuration like '%\\\"project\\\": \\\"cmip5\\\"%' and timestamp between '2020-01-01' and '2021-04-19'\"\n",
"record.execute(query)\n",
"result=record.fetchall()\n",
"plugin=np.array(result)\n",
"##### Print plugin names and its count####\n",
"for key,value in dict(Counter(plugin.ravel())).items():\n",
" print(key+\"(\"+str(value)+\")\")\n",
"~~~"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (using the module python/3.5.2)",
"language": "python",
"name": "python_3.5.2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment