Skip to content

Commit

Permalink
Template class (#463)
Browse files Browse the repository at this point in the history
* savepoint in creating a template class

* savepoint 2

* savepoint

* Move to json file for template

* savepoint

* working on demo dataset savepoint

* minor fixes in template for wide data cases

* bugfix on required metadf templates

* Add compatibility and blacklist checsk

* get IO test running

* fix breaking test

* fix plot test

* Working analysis as inheritance of _datasetbase

* fix doc example

* minor fixes on tests

* resturcture deploy pipeline

* cleanup of logs

* fix descrip error for 'T' to 'min'

* savepoint on getting doctest to work

* doctest are working !!

* update general pages in the documentation

* testing the template class on mulitple datasets

* test on amsterdam data

* cleanup an depric error fixes

* bump version

* code review and doc update to ipynb

* Saved new PDF of paper

* station class based on abstract attributes setters

* add a current data unit attribute for modeldata obstypes to fix unit conversion after saving modeldata to pkl

* bugfix: add current units to import from csv file

---------

Co-authored-by: vergauwenthomas <vergauwenthomas@users.noreply.github.com>
vergauwenthomas and vergauwenthomas authored Jul 16, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent f603457 commit d31e358
Showing 76 changed files with 23,476 additions and 2,150 deletions.
7 changes: 2 additions & 5 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -10,12 +10,9 @@ poetry.lock
.virtual_documents

#Exclude deploy scripts
deploiment/*_log
deploiment/*online* #only for thomas
deploiment/update_version.sh #only for thomas
deploiment/*
deploiment/logs/*

#Exclude local package builds
#Exclude local package builds (are build on github)
dist/*

#exclude the python versions of the doc example notebooks
127 changes: 0 additions & 127 deletions deploiment/build_and_test.sh

This file was deleted.

30 changes: 30 additions & 0 deletions deploiment/build_documentation.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env bash

# This script will build the documentation of the MetObs toolkit

echo " ---- Building Documentation ----"

DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

cd ${DEPLOY_DIR}
cd .. #Navigate to workdir
WORKDIR=$(pwd)



#1. Setup environment
poetry install --all-extras

#2. Cleanup builds
DOCDIR=${WORKDIR}/docs
#clear builds

rm -r ${DOCDIR}/_build/*
rm -r ${DOCDIR}/_autosummary/*

#3. Build documentation
cd ${WORKDIR}
sphinx-build -a -E -v docs/ docs/_build/

#4. Back to deploy dir
cd ${DEPLOY_DIR}
61 changes: 61 additions & 0 deletions deploiment/build_package.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/usr/bin/env bash

# This script will build the package and updates dependencies by updateing the pyproject.toml file of the MetObs toolkit

echo " ---- Building and Updating Metobs Toolkit ----"

DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

cd ${DEPLOY_DIR}
cd .. #Navigate to workdir
WORKDIR=$(pwd)

DISTDIR=${WORKDIR}/dist


#1. cleanup previous builds
rm ${DISTDIR}/*.whl
rm ${DISTDIR}/*.tar.gz

#2. Update the dependencies in the toml
poetry update

# Toolkit dependencies
poetry add numpy@^1 #v2.x.x conflicting with titanlib
poetry add cartopy@latest
poetry add earthengine-api@latest
poetry add geemap@latest
poetry add geopandas@latest
poetry add geos@latest
poetry add mapclassify@latest
poetry add matplotlib@latest
poetry add pandas@^2
poetry add shapely@latest


# Toolkit DEV group
poetry add poetry@latest --group dev
poetry add pre-commit@latest --group dev

# Toolkit documentation group
poetry add myst_parser@latest --group documentation
poetry add nbsphinx@latest --group documentation
poetry add pandoc@latest --group documentation
poetry add pydata-sphinx-theme@latest --group documentation
poetry add sphinx@latest --group documentation
poetry add sphinx-copybutton@latest --group documentation
poetry add sphinx-rtd-theme@latest --group documentation

# Toolkit titan group
poetry add titanlib@latest --group titan


# 3. update the lock file
#poetry update #to update the poetry.lock with the latest versions of the depending packages
poetry install --all-extras
poetry show


# 4. Make a new build
poetry build
cd ${DEPLOY_DIR}
33 changes: 33 additions & 0 deletions deploiment/develop_pipeline.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash

# Run a full develop pipeline on the toolkit (update + build + doc + tests)

echo " -----------------------------------"
echo " ---- Running DEV pipeline ----"
echo " -----------------------------------"


DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

cd ${DEPLOY_DIR}

#1. Update and built package
./build_package.sh


#remove previous logs
LOGDIR=${DEPLOY_DIR}/logs
rm ${LOGDIR}/*_log

#2. Run examples as test
./run_examples_as_test.sh

#3. Build documentation
./build_documentation.sh


#4. Run toolkit tests
./run_tests.sh

#5. Run Docstring tests
./run_doctests.sh
33 changes: 33 additions & 0 deletions deploiment/run_doctests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash

# This script will run the packgage tests

echo " ---- Running the toolkit-DOC-tests ----"

DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

cd ${DEPLOY_DIR}
cd .. #Navigate to workdir
WORKDIR=$(pwd)
LOGDIR=${DEPLOY_DIR}/logs

rm ${LOGDIR}/DOCtest_output_*_log



#Run Doctests on all modules
echo "RUNNING DOCTESTS NOW"
cd ${WORKDIR}/metobs_toolkit
modules=`ls ./*.py`
cd ${WORKDIR} #call poetry run from in root?
for t in $modules; do
module_file=${WORKDIR}/metobs_toolkit/${t}
echo "Running doctests in ${module_file}"

python3 -m doctest -o ELLIPSIS -o NORMALIZE_WHITESPACE ${module_file} | tee ${LOGDIR}/DOCtest_output_${t:2:-3}_log
done

rm ${WORKDIR}/*.pkl #created by doctest
rm ${WORKDIR}/*.csv #created by docstest

cd ${DEPLOY_DIR}
78 changes: 78 additions & 0 deletions deploiment/run_examples_as_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/usr/bin/env bash

# This script will run the notebook examples and documentation notebooks as tests

echo " ---- Running notebook examples (from docs) as tests ----"

DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

cd ${DEPLOY_DIR}
cd .. #Navigate to workdir
WORKDIR=$(pwd)
LOGDIR=${DEPLOY_DIR}/logs
DOCEXAMPLEDIR=${WORKDIR}/docs/examples
OTHER_NOTEBOOKS=${WORKDIR}/docs/notebook_references


#make logfile for each test and stream prompt output for the test
make_test_log () {
testfile="$1"
specificlogfile="${LOGDIR}/${testfile::-3}_log"
rm -f ${specificlogfile}
touch ${specificlogfile}
echo "$specificlogfile"
}


#1. Remove all .py files
#delete all .py versions of the examples (rebuild them from the notebooks)
rm ${DOCEXAMPLEDIR}/*.py
rm ${OTHER_NOTEBOOKS}/*.py

#2. Convert the notebooks back to .py files
#convert nb to python files
cd ${DOCEXAMPLEDIR}
jupyter nbconvert --to python *.ipynb

cd ${OTHER_NOTEBOOKS}
jupyter nbconvert --to python *.ipynb

#3. Run examples
cd ${DOCEXAMPLEDIR}
filenames=`ls ./*.py`
for t in $filenames; do
example_file=${DOCEXAMPLEDIR}/${t}
logfile="$(make_test_log ${t})"
echo Running ${t} as a test
poetry run python ${example_file} >> ${logfile} 2>&1
if [ $? -eq 0 ]; then
echo "succeeded !!"
else
echo "FAIL!!"
fi

done

#3. Run other notebooks used in the documentation
cd ${OTHER_NOTEBOOKS}
filenames=`ls ./*.py`
for t in $filenames; do
example_file=${OTHER_NOTEBOOKS}/${t}
logfile="$(make_test_log ${t})"
echo Running ${t} as a test
poetry run python ${example_file} >> ${logfile} 2>&1
if [ $? -eq 0 ]; then
echo "succeeded !!"
else
echo "FAIL!!"
fi

done






# Back to deploy
cd ${DEPLOY_DIR}
Loading

0 comments on commit d31e358

Please sign in to comment.