Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[pre-commit.ci] pre-commit autoupdate #130

Merged
merged 2 commits into from
Sep 3, 2024
Merged

[pre-commit.ci] pre-commit autoupdate #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
69c4e57
[pre-commit.ci] pre-commit autoupdate
pre-commit-ci[bot] Sep 2, 2024
640c295
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Sep 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ repos:
- id: add-trailing-comma

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.5.6
rev: v0.6.3
hooks:
- id: ruff
args: ["--fix", "--show-fixes"]
Expand Down
2 changes: 1 addition & 1 deletion docs/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ ipykernel
matplotlib>=3
nbsphinx
pocean-core>=3
pooch
pynco
sphinx
sphinx-autodoc-typehints
pooch
69 changes: 41 additions & 28 deletions docs/source/examples/QartodTestExample_Glider.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@
"metadata": {},
"outputs": [],
"source": [
"import ioos_qc\n",
"from ioos_qc.config import Config\n",
"from ioos_qc.qartod import aggregate\n",
"from ioos_qc.streams import XarrayStream\n",
"from ioos_qc.results import collect_results, CollectedResult\n",
"import xarray as xr\n",
"import numpy as np\n",
"import pandas as pd\n",
"import pooch\n",
"import xarray as xr\n",
"from bokeh.plotting import output_notebook\n",
"\n",
"from ioos_qc.config import Config\n",
"from ioos_qc.qartod import aggregate\n",
"from ioos_qc.results import CollectedResult, collect_results\n",
"from ioos_qc.streams import XarrayStream\n",
"\n",
"output_notebook()"
]
},
Expand All @@ -46,7 +46,7 @@
"metadata": {},
"outputs": [],
"source": [
"url = f\"https://github.com/ioos/ioos_qc/releases/download\"\n",
"url = \"https://github.com/ioos/ioos_qc/releases/download\"\n",
"version = \"2.1.0\"\n",
"fname = \"nrt_SEA067_M37.nc\"\n",
"\n",
Expand Down Expand Up @@ -75,21 +75,21 @@
" \"temperature\": {\n",
" \"qartod\": {\n",
" \"gross_range_test\": {\"suspect_span\": [0, 30], \"fail_span\": [-2.5, 40]},\n",
" \"spike_test\": {\"suspect_threshold\": 2.0, \"fail_threshold\": 6.0}\n",
" }\n",
" \"spike_test\": {\"suspect_threshold\": 2.0, \"fail_threshold\": 6.0},\n",
" },\n",
" },\n",
" \"conductivity\": {\n",
" \"qartod\": {\n",
" \"gross_range_test\": {\"suspect_span\": [6, 42], \"fail_span\": [3, 45]}\n",
" }\n",
" \"gross_range_test\": {\"suspect_span\": [6, 42], \"fail_span\": [3, 45]},\n",
" },\n",
" },\n",
" \"salinity\": {\n",
" \"qartod\": {\n",
" \"gross_range_test\": {\"suspect_span\": [5, 38], \"fail_span\": [2, 41]},\n",
" \"spike_test\": {\"suspect_threshold\": 0.3, \"fail_threshold\": 0.9},\n",
" \"location_test\": {\"bbox\": [10, 50, 25, 60]},\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}"
]
},
Expand All @@ -110,7 +110,7 @@
"c = Config(config)\n",
"qc = XarrayStream(ds, lon=\"longitude\", lat=\"latitude\")\n",
"runner = list(qc.run(c))\n",
"results = collect_results(runner, how='list')\n"
"results = collect_results(runner, how=\"list\")"
]
},
{
Expand All @@ -129,13 +129,13 @@
"outputs": [],
"source": [
"agg = CollectedResult(\n",
" stream_id='',\n",
" package='qartod',\n",
" test='qc_rollup',\n",
" stream_id=\"\",\n",
" package=\"qartod\",\n",
" test=\"qc_rollup\",\n",
" function=aggregate,\n",
" results=aggregate(results),\n",
" tinp=qc.time(),\n",
" data=ds\n",
" data=ds,\n",
")\n",
"flag_vals = agg.results"
]
Expand All @@ -160,7 +160,15 @@
"meaning[flag_vals == 9] = \"MISSING\"\n",
"meaning[flag_vals == 3] = \"SUSPECT\"\n",
"meaning[flag_vals == 4] = \"FAIL\"\n",
"df = pd.DataFrame({\"time\": time, \"salinity\": ds[\"salinity\"], \"flag\": flag_vals, \"depth\": ds.depth, \"quality control\": meaning})"
"df = pd.DataFrame(\n",
" {\n",
" \"time\": time,\n",
" \"salinity\": ds[\"salinity\"],\n",
" \"flag\": flag_vals,\n",
" \"depth\": ds.depth,\n",
" \"quality control\": meaning,\n",
" }\n",
")"
]
},
{
Expand All @@ -170,19 +178,24 @@
"outputs": [],
"source": [
"from bokeh.plotting import figure, show\n",
"from bokeh.sampledata.penguins import data\n",
"from bokeh.transform import factor_cmap, factor_mark\n",
"\n",
"flag_vals = [\"GOOD\",\"UNKNOWN\",\"MISSING\",\"SUSPECT\",\"FAIL\"]\n",
"markers = ['hex', 'circle_x', 'circle', 'triangle', 'square']\n",
"flag_vals = [\"GOOD\", \"UNKNOWN\", \"MISSING\", \"SUSPECT\", \"FAIL\"]\n",
"markers = [\"hex\", \"circle_x\", \"circle\", \"triangle\", \"square\"]\n",
"\n",
"p = figure(title = \"Salinity flags\", background_fill_color=\"#fafafa\", x_axis_type='datetime')\n",
"p.yaxis.axis_label = 'salinity (PSU)'\n",
"p = figure(title=\"Salinity flags\", background_fill_color=\"#fafafa\", x_axis_type=\"datetime\")\n",
"p.yaxis.axis_label = \"salinity (PSU)\"\n",
"\n",
"p.scatter(\"time\", \"salinity\", source=df,\n",
" legend_group=\"quality control\", fill_alpha=0.4, size=12,\n",
" marker=factor_mark('quality control', markers, flag_vals),\n",
" color=factor_cmap('quality control', 'Category10_5', flag_vals))\n",
"p.scatter(\n",
" \"time\",\n",
" \"salinity\",\n",
" source=df,\n",
" legend_group=\"quality control\",\n",
" fill_alpha=0.4,\n",
" size=12,\n",
" marker=factor_mark(\"quality control\", markers, flag_vals),\n",
" color=factor_cmap(\"quality control\", \"Category10_5\", flag_vals),\n",
")\n",
"\n",
"p.legend.location = \"top_left\"\n",
"p.legend.title = \"IOOS flags\"\n",
Expand Down
67 changes: 40 additions & 27 deletions docs/source/examples/QartodTestExample_SalinityAttenuation.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
"\n",
"\n",
"def plot_results(data, var_name, results, title, test_name):\n",
"\n",
" time = data.index\n",
" obs = data[var_name]\n",
" qc_test = results[\"qartod\"][test_name]\n",
Expand All @@ -61,15 +60,30 @@
"\n",
" p1.line(time, obs, legend_label=\"obs\", color=\"#A6CEE3\")\n",
" p1.circle(\n",
" time, qc_notrun, size=2, legend_label=\"qc not run\", color=\"gray\", alpha=0.2\n",
" time,\n",
" qc_notrun,\n",
" size=2,\n",
" legend_label=\"qc not run\",\n",
" color=\"gray\",\n",
" alpha=0.2,\n",
" )\n",
" p1.circle(time, qc_pass, size=4, legend_label=\"qc pass\", color=\"green\", alpha=0.5)\n",
" p1.circle(\n",
" time, qc_suspect, size=4, legend_label=\"qc suspect\", color=\"orange\", alpha=0.7\n",
" time,\n",
" qc_suspect,\n",
" size=4,\n",
" legend_label=\"qc suspect\",\n",
" color=\"orange\",\n",
" alpha=0.7,\n",
" )\n",
" p1.circle(time, qc_fail, size=6, legend_label=\"qc fail\", color=\"red\", alpha=1.0)\n",
" p1.circle(\n",
" time, qc_notrun, size=6, legend_label=\"qc not eval\", color=\"gray\", alpha=1.0\n",
" time,\n",
" qc_notrun,\n",
" size=6,\n",
" legend_label=\"qc not eval\",\n",
" color=\"gray\",\n",
" alpha=1.0,\n",
" )\n",
"\n",
" plotting.show(gridplot([[p1]], width=800, height=400))"
Expand All @@ -94,7 +108,6 @@
"source": [
"import pandas as pd\n",
"\n",
"\n",
"url = \"https://github.com/ioos/ioos_qc/raw/master/docs/source/examples\"\n",
"fname = f\"{url}/attenuated_salinity_example.csv\"\n",
"\n",
Expand Down Expand Up @@ -156,7 +169,8 @@
"outputs": [],
"source": [
"range_data = data.rolling(f\"{time_delta}S\", min_periods=int(min_period_obs)).apply(\n",
" np.ptp, raw=True\n",
" np.ptp,\n",
" raw=True,\n",
")\n",
"\n",
"range_data.plot();"
Expand Down Expand Up @@ -194,7 +208,8 @@
"outputs": [],
"source": [
"stdev_data = data.rolling(f\"{time_delta}S\", min_periods=int(min_period_obs)).apply(\n",
" np.std, raw=True\n",
" np.std,\n",
" raw=True,\n",
")\n",
"\n",
"stdev_data.plot();"
Expand All @@ -219,10 +234,8 @@
"metadata": {},
"outputs": [],
"source": [
"import ioos_qc\n",
"from ioos_qc.config import QcConfig\n",
"\n",
"\n",
"qc_config = {\n",
" \"qartod\": {\n",
" \"attenuated_signal_test\": {\n",
Expand All @@ -231,8 +244,8 @@
" \"test_period\": int(time_delta),\n",
" \"min_period\": min_period_secs,\n",
" \"check_type\": \"range\",\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down Expand Up @@ -268,8 +281,8 @@
" \"check_type\": \"std\",\n",
" \"test_period\": int(time_delta),\n",
" \"min_period\": min_period_secs,\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down Expand Up @@ -309,8 +322,8 @@
" \"check_type\": \"std\",\n",
" \"test_period\": int(time_delta),\n",
" \"min_period\": 0,\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down Expand Up @@ -344,8 +357,8 @@
" \"check_type\": \"std\",\n",
" \"test_period\": int(time_delta),\n",
" \"min_period\": 10 * 60,\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down Expand Up @@ -375,8 +388,8 @@
" \"check_type\": \"std\",\n",
" \"test_period\": int(time_delta),\n",
" \"min_period\": 60 * 60,\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down Expand Up @@ -408,8 +421,8 @@
" \"check_type\": \"std\",\n",
" \"test_period\": int(time_delta),\n",
" \"min_period\": int(time_delta / 2),\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down Expand Up @@ -442,8 +455,8 @@
" \"check_type\": \"std\",\n",
" \"test_period\": int(time_delta),\n",
" \"min_period\": min_period_secs,\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down Expand Up @@ -473,8 +486,8 @@
" \"check_type\": \"std\",\n",
" \"test_period\": int(time_delta),\n",
" \"min_period\": min_period_secs,\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down Expand Up @@ -504,8 +517,8 @@
" \"check_type\": \"std\",\n",
" \"test_period\": int(time_delta),\n",
" \"min_period\": min_period_secs,\n",
" }\n",
" }\n",
" },\n",
" },\n",
"}\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[var_name], tinp=data.index.values)\n",
Expand Down
18 changes: 13 additions & 5 deletions docs/source/examples/QartodTestExample_WaterLevel.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
"\n",
"\n",
"def plot_results(data, var_name, results, title, test_name):\n",
"\n",
" time = data[\"time\"]\n",
" obs = data[var_name]\n",
" qc_test = results[\"qartod\"][test_name]\n",
Expand All @@ -47,11 +46,21 @@
"\n",
" p1.line(time, obs, legend_label=\"obs\", color=\"#A6CEE3\")\n",
" p1.circle(\n",
" time, qc_notrun, size=2, legend_label=\"qc not run\", color=\"gray\", alpha=0.2\n",
" time,\n",
" qc_notrun,\n",
" size=2,\n",
" legend_label=\"qc not run\",\n",
" color=\"gray\",\n",
" alpha=0.2,\n",
" )\n",
" p1.circle(time, qc_pass, size=4, legend_label=\"qc pass\", color=\"green\", alpha=0.5)\n",
" p1.circle(\n",
" time, qc_suspect, size=4, legend_label=\"qc suspect\", color=\"orange\", alpha=0.7\n",
" time,\n",
" qc_suspect,\n",
" size=4,\n",
" legend_label=\"qc suspect\",\n",
" color=\"orange\",\n",
" alpha=0.7,\n",
" )\n",
" p1.circle(time, qc_fail, size=6, legend_label=\"qc fail\", color=\"red\", alpha=1.0)\n",
"\n",
Expand Down Expand Up @@ -86,7 +95,7 @@
" },\n",
" \"rate_of_change_test\": {\"threshold\": 0.001},\n",
" \"spike_test\": {\"suspect_threshold\": 0.8, \"fail_threshold\": 3},\n",
" }\n",
" },\n",
"}"
]
},
Expand Down Expand Up @@ -121,7 +130,6 @@
"source": [
"from ioos_qc.config import QcConfig\n",
"\n",
"\n",
"qc = QcConfig(qc_config)\n",
"qc_results = qc.run(inp=data[variable_name], tinp=data[\"timestamp\"], zinp=data[\"z\"])\n",
"qc_results"
Expand Down
Loading