-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Loading status checks…
Template class (#463)
* savepoint in creating a template class * savepoint 2 * savepoint * Move to json file for template * savepoint * working on demo dataset savepoint * minor fixes in template for wide data cases * bugfix on required metadf templates * Add compatibility and blacklist checsk * get IO test running * fix breaking test * fix plot test * Working analysis as inheritance of _datasetbase * fix doc example * minor fixes on tests * resturcture deploy pipeline * cleanup of logs * fix descrip error for 'T' to 'min' * savepoint on getting doctest to work * doctest are working !! * update general pages in the documentation * testing the template class on mulitple datasets * test on amsterdam data * cleanup an depric error fixes * bump version * code review and doc update to ipynb * Saved new PDF of paper * station class based on abstract attributes setters * add a current data unit attribute for modeldata obstypes to fix unit conversion after saving modeldata to pkl * bugfix: add current units to import from csv file --------- Co-authored-by: vergauwenthomas <vergauwenthomas@users.noreply.github.com>
1 parent
f603457
commit d31e358
Showing
76 changed files
with
23,476 additions
and
2,150 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
#!/usr/bin/env bash | ||
|
||
# This script will build the documentation of the MetObs toolkit | ||
|
||
echo " ---- Building Documentation ----" | ||
|
||
DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) | ||
|
||
cd ${DEPLOY_DIR} | ||
cd .. #Navigate to workdir | ||
WORKDIR=$(pwd) | ||
|
||
|
||
|
||
#1. Setup environment | ||
poetry install --all-extras | ||
|
||
#2. Cleanup builds | ||
DOCDIR=${WORKDIR}/docs | ||
#clear builds | ||
|
||
rm -r ${DOCDIR}/_build/* | ||
rm -r ${DOCDIR}/_autosummary/* | ||
|
||
#3. Build documentation | ||
cd ${WORKDIR} | ||
sphinx-build -a -E -v docs/ docs/_build/ | ||
|
||
#4. Back to deploy dir | ||
cd ${DEPLOY_DIR} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
#!/usr/bin/env bash | ||
|
||
# This script will build the package and updates dependencies by updateing the pyproject.toml file of the MetObs toolkit | ||
|
||
echo " ---- Building and Updating Metobs Toolkit ----" | ||
|
||
DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) | ||
|
||
cd ${DEPLOY_DIR} | ||
cd .. #Navigate to workdir | ||
WORKDIR=$(pwd) | ||
|
||
DISTDIR=${WORKDIR}/dist | ||
|
||
|
||
#1. cleanup previous builds | ||
rm ${DISTDIR}/*.whl | ||
rm ${DISTDIR}/*.tar.gz | ||
|
||
#2. Update the dependencies in the toml | ||
poetry update | ||
|
||
# Toolkit dependencies | ||
poetry add numpy@^1 #v2.x.x conflicting with titanlib | ||
poetry add cartopy@latest | ||
poetry add earthengine-api@latest | ||
poetry add geemap@latest | ||
poetry add geopandas@latest | ||
poetry add geos@latest | ||
poetry add mapclassify@latest | ||
poetry add matplotlib@latest | ||
poetry add pandas@^2 | ||
poetry add shapely@latest | ||
|
||
|
||
# Toolkit DEV group | ||
poetry add poetry@latest --group dev | ||
poetry add pre-commit@latest --group dev | ||
|
||
# Toolkit documentation group | ||
poetry add myst_parser@latest --group documentation | ||
poetry add nbsphinx@latest --group documentation | ||
poetry add pandoc@latest --group documentation | ||
poetry add pydata-sphinx-theme@latest --group documentation | ||
poetry add sphinx@latest --group documentation | ||
poetry add sphinx-copybutton@latest --group documentation | ||
poetry add sphinx-rtd-theme@latest --group documentation | ||
|
||
# Toolkit titan group | ||
poetry add titanlib@latest --group titan | ||
|
||
|
||
# 3. update the lock file | ||
#poetry update #to update the poetry.lock with the latest versions of the depending packages | ||
poetry install --all-extras | ||
poetry show | ||
|
||
|
||
# 4. Make a new build | ||
poetry build | ||
cd ${DEPLOY_DIR} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
#!/usr/bin/env bash | ||
|
||
# Run a full develop pipeline on the toolkit (update + build + doc + tests) | ||
|
||
echo " -----------------------------------" | ||
echo " ---- Running DEV pipeline ----" | ||
echo " -----------------------------------" | ||
|
||
|
||
DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) | ||
|
||
cd ${DEPLOY_DIR} | ||
|
||
#1. Update and built package | ||
./build_package.sh | ||
|
||
|
||
#remove previous logs | ||
LOGDIR=${DEPLOY_DIR}/logs | ||
rm ${LOGDIR}/*_log | ||
|
||
#2. Run examples as test | ||
./run_examples_as_test.sh | ||
|
||
#3. Build documentation | ||
./build_documentation.sh | ||
|
||
|
||
#4. Run toolkit tests | ||
./run_tests.sh | ||
|
||
#5. Run Docstring tests | ||
./run_doctests.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
#!/usr/bin/env bash | ||
|
||
# This script will run the packgage tests | ||
|
||
echo " ---- Running the toolkit-DOC-tests ----" | ||
|
||
DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) | ||
|
||
cd ${DEPLOY_DIR} | ||
cd .. #Navigate to workdir | ||
WORKDIR=$(pwd) | ||
LOGDIR=${DEPLOY_DIR}/logs | ||
|
||
rm ${LOGDIR}/DOCtest_output_*_log | ||
|
||
|
||
|
||
#Run Doctests on all modules | ||
echo "RUNNING DOCTESTS NOW" | ||
cd ${WORKDIR}/metobs_toolkit | ||
modules=`ls ./*.py` | ||
cd ${WORKDIR} #call poetry run from in root? | ||
for t in $modules; do | ||
module_file=${WORKDIR}/metobs_toolkit/${t} | ||
echo "Running doctests in ${module_file}" | ||
|
||
python3 -m doctest -o ELLIPSIS -o NORMALIZE_WHITESPACE ${module_file} | tee ${LOGDIR}/DOCtest_output_${t:2:-3}_log | ||
done | ||
|
||
rm ${WORKDIR}/*.pkl #created by doctest | ||
rm ${WORKDIR}/*.csv #created by docstest | ||
|
||
cd ${DEPLOY_DIR} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
#!/usr/bin/env bash | ||
|
||
# This script will run the notebook examples and documentation notebooks as tests | ||
|
||
echo " ---- Running notebook examples (from docs) as tests ----" | ||
|
||
DEPLOY_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) | ||
|
||
cd ${DEPLOY_DIR} | ||
cd .. #Navigate to workdir | ||
WORKDIR=$(pwd) | ||
LOGDIR=${DEPLOY_DIR}/logs | ||
DOCEXAMPLEDIR=${WORKDIR}/docs/examples | ||
OTHER_NOTEBOOKS=${WORKDIR}/docs/notebook_references | ||
|
||
|
||
#make logfile for each test and stream prompt output for the test | ||
make_test_log () { | ||
testfile="$1" | ||
specificlogfile="${LOGDIR}/${testfile::-3}_log" | ||
rm -f ${specificlogfile} | ||
touch ${specificlogfile} | ||
echo "$specificlogfile" | ||
} | ||
|
||
|
||
#1. Remove all .py files | ||
#delete all .py versions of the examples (rebuild them from the notebooks) | ||
rm ${DOCEXAMPLEDIR}/*.py | ||
rm ${OTHER_NOTEBOOKS}/*.py | ||
|
||
#2. Convert the notebooks back to .py files | ||
#convert nb to python files | ||
cd ${DOCEXAMPLEDIR} | ||
jupyter nbconvert --to python *.ipynb | ||
|
||
cd ${OTHER_NOTEBOOKS} | ||
jupyter nbconvert --to python *.ipynb | ||
|
||
#3. Run examples | ||
cd ${DOCEXAMPLEDIR} | ||
filenames=`ls ./*.py` | ||
for t in $filenames; do | ||
example_file=${DOCEXAMPLEDIR}/${t} | ||
logfile="$(make_test_log ${t})" | ||
echo Running ${t} as a test | ||
poetry run python ${example_file} >> ${logfile} 2>&1 | ||
if [ $? -eq 0 ]; then | ||
echo "succeeded !!" | ||
else | ||
echo "FAIL!!" | ||
fi | ||
|
||
done | ||
|
||
#3. Run other notebooks used in the documentation | ||
cd ${OTHER_NOTEBOOKS} | ||
filenames=`ls ./*.py` | ||
for t in $filenames; do | ||
example_file=${OTHER_NOTEBOOKS}/${t} | ||
logfile="$(make_test_log ${t})" | ||
echo Running ${t} as a test | ||
poetry run python ${example_file} >> ${logfile} 2>&1 | ||
if [ $? -eq 0 ]; then | ||
echo "succeeded !!" | ||
else | ||
echo "FAIL!!" | ||
fi | ||
|
||
done | ||
|
||
|
||
|
||
|
||
|
||
|
||
# Back to deploy | ||
cd ${DEPLOY_DIR} |
Oops, something went wrong.