diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..0678e48561e38e9fb247808b295fd5232a323349 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +*.DS* +.vscode +.idea +.ipynb_checkpoints +*/.ipynb_checkpoints +*/*/.ipynb_checkpoints +.vscode +*.log +.idea* +conda_env/ +**/*.sw[pco] +**/*.*.sw[pco] +**/.ipynb_checkpoints* +**/*.py[co] +**/*~ +**/._README.md +**/*egg* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..600701606783de5e5559d53aad2602fc1d7fef98 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +# makefile used for testing +# +# +ENV_NAME=era5-tables +.PHONY: all conda install test + +all: install test + +conda: + conda create -n $(ENV_NAME) python=3.11 + +install: conda + @# python3 -m pip install -e .[dev] + conda run -n $(ENV_NAME) python -m pip install -e .[dev] + +check: + conda activate $(ENV_NAME); cfchecks $(file) + +clean: + conda remove -n $(ENV_NAME) --all -y + +lint: + mypy --install-types --non-interactive + black --check -t py311 -l 79 src + flake8 src/rechunk_data --count --max-complexity=15 --max-line-length=88 --statistics --doctests \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..ea16802f7f92a2c1e1ce8540ed9d8f57a09eb2c9 --- /dev/null +++ b/setup.py @@ -0,0 +1,25 @@ +from setuptools import setup, find_packages + +setup( + name='era5-tables', + version='0.1', + packages=find_packages(where='src'), + package_dir={'': 'src'}, + install_requires=[ + 'pandas>=2.0.0', + 'openpyxl>=3.1.0', + 'cfchecker>=4.1.0', + 'netCDF4>=1.6.0', + 'numpy>=1.20.0' + ], + extras_require={ + 'dev': [ + 'pytest', + 'mypy', + 'black', + 'flake8', + 'ipython' + ] + }, + python_requires='>=3.9', +) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/converter.py b/src/converter.py new file mode 100644 index 0000000000000000000000000000000000000000..803128db76882f8af9dd9c021c158e1e0ac0ea72 --- /dev/null +++ b/src/converter.py @@ -0,0 +1,19 @@ +import pandas as pd +import json +import csv +from pathlib import Path + +def excel_to_csv(excel_path, csv_path, sheet_name=0, field_separator="|"): + df = pd.read_excel(excel_path, sheet_name=sheet_name) + df.to_csv(csv_path, sep=field_separator, index=False) + return csv_path + +def csv_to_json(csv_path, json_path): + with open(csv_path, encoding='utf-8') as csvf: + csv_reader = csv.DictReader(csvf) + data = [row for row in csv_reader] + + with open(json_path, 'w', encoding='utf-8') as jsonf: + json.dump(data, jsonf, indent=2) + + return json_path diff --git a/src/validator.py b/src/validator.py new file mode 100644 index 0000000000000000000000000000000000000000..760b7e64111bf504acada1e8af57be5351ab2cc6 --- /dev/null +++ b/src/validator.py @@ -0,0 +1,10 @@ +import subprocess +import sys + +def cf_check(nc_file): + try: + result = subprocess.run(['cfchecks', nc_file], capture_output=True, text=True, check=True) + print(result.stdout) + except subprocess.CalledProcessError as e: + print("CF checker error:", e.stderr, file=sys.stderr) + sys.exit(1)