From dabb7280b4337e6a8cc3cd1d8bdd530500353cd7 Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Tue, 5 Dec 2023 09:56:44 -0800 Subject: [PATCH] Update regression test notebook to show validation of all vars --- .../12-4-23-qa-no-cdms-slice.ipynb | 811 ++++++++++++++---- 1 file changed, 651 insertions(+), 160 deletions(-) diff --git a/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb b/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb index 7f53b053c..7a8d8ebae 100644 --- a/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb +++ b/auxiliary_tools/cdat_regression_testing/671-lat-lon/12-4-23-qa-no-cdms-slice.ipynb @@ -20,26 +20,19 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ + "import pandas as pd\n", + "\n", "import glob\n", - "from auxiliary_tools.cdat_regression_testing.utils import (\n", - " get_metrics,\n", - " get_rel_diffs,\n", - " get_num_metrics_above_diff_thres,\n", - " highlight_large_diffs,\n", - " sort_columns,\n", - " update_diffs_to_pct,\n", - " PERCENTAGE_COLUMNS,\n", - ")\n", + "from auxiliary_tools.cdat_regression_testing.utils import get_metrics, get_rel_diffs\n", "\n", - "import pandas as pd\n", "\n", "# TODO: Update DEV_RESULTS and MAIN_RESULTS to your diagnostic sets.\n", "DEV_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_658/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n", - "MAIN_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_main_no_slice/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n", + "MAIN_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/examples_main_w_slice/ex1_modTS_vs_modTS_3years/lat_lon/model_vs_model\"\n", "\n", "DEV_GLOB = sorted(glob.glob(DEV_PATH + \"/*.json\"))\n", "MAIN_GLOB = sorted(glob.glob(MAIN_PATH + \"/*.json\"))" @@ -54,26 +47,19 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "df_metrics_dev = get_metrics(DEV_GLOB)\n", "df_metrics_main = get_metrics(MAIN_GLOB)\n", "\n", - "df_metrics_dev2 = df_metrics_dev.reset_index(names=[\"var_key\", \"metric\"])\n", - "df_metrics_dev2 = df_metrics_dev2.loc[\n", - " df_metrics_dev2.var_key.isin(df_metrics_main.index.get_level_values(0).unique())\n", - "]\n", - "df_metrics_dev2 = df_metrics_dev2.set_index([\"var_key\", \"metric\"])\n", - "\n", - "\n", - "df_metrics_diffs = get_rel_diffs(df_metrics_dev2, df_metrics_main)" + "df_metrics_diffs = get_rel_diffs(df_metrics_dev, df_metrics_main)" ] }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -105,19 +91,229 @@ " diff DIFF (%)\n", " misc DIFF (%)\n", " \n", - " \n", - " var_key\n", - " metric\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", " \n", + " FLNS\n", + " min\n", + " 1.317678e-16\n", + " 0.000000e+00\n", + " 1.317678e-16\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " NaN\n", + " \n", + " \n", + " max\n", + " 0.000000e+00\n", + " 2.058062e-16\n", + " 0.000000e+00\n", + " 2.058062e-16\n", + " 7.203322e-16\n", + " NaN\n", + " \n", + " \n", + " mean\n", + " 5.322645e-16\n", + " 1.276503e-15\n", + " 5.322645e-16\n", + " 1.276503e-15\n", + " 1.764203e-15\n", + " NaN\n", + " \n", + " \n", + " std\n", + " NaN\n", + " NaN\n", + " 5.455412e-16\n", + " 5.344197e-16\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " rmse\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " corr\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " FLUT\n", + " min\n", + " 1.140418e-16\n", + " 0.000000e+00\n", + " 1.140418e-16\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " NaN\n", + " \n", + " \n", + " max\n", + " 0.000000e+00\n", + " 1.893757e-16\n", + " 0.000000e+00\n", + " 1.893757e-16\n", + " 0.000000e+00\n", + " NaN\n", + " \n", + " \n", + " mean\n", + " 1.779953e-15\n", + " 1.764910e-15\n", + " 1.779953e-15\n", + " 1.764910e-15\n", + " 1.522792e-15\n", + " NaN\n", + " \n", + " \n", + " std\n", + " NaN\n", + " NaN\n", + " 2.315874e-16\n", + " 4.524283e-16\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " rmse\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 2.566283e-16\n", + " \n", + " \n", + " corr\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " FSNS\n", + " min\n", + " 2.102518e-16\n", + " 4.252167e-16\n", + " 2.102518e-16\n", + " 4.252167e-16\n", + " 9.861022e-16\n", + " NaN\n", + " \n", + " \n", + " max\n", + " 0.000000e+00\n", + " 2.084296e-16\n", + " 0.000000e+00\n", + " 2.084296e-16\n", + " 3.441230e-16\n", + " NaN\n", + " \n", + " \n", + " mean\n", + " 2.107767e-15\n", + " 1.718080e-15\n", + " 2.107767e-15\n", + " 1.718080e-15\n", + " 1.350994e-15\n", + " NaN\n", + " \n", + " \n", + " std\n", + " NaN\n", + " NaN\n", + " 7.842697e-16\n", + " 2.191950e-16\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " rmse\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " corr\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " FSNTOA\n", + " min\n", + " 1.582252e-16\n", + " 0.000000e+00\n", + " 1.582252e-16\n", + " 0.000000e+00\n", + " 1.205526e-15\n", + " NaN\n", + " \n", + " \n", + " max\n", + " 0.000000e+00\n", + " 1.569442e-16\n", + " 0.000000e+00\n", + " 1.569442e-16\n", + " 1.527862e-15\n", + " NaN\n", + " \n", + " \n", + " mean\n", + " 8.294511e-16\n", + " 9.417413e-16\n", + " 8.294511e-16\n", + " 9.417413e-16\n", + " 1.405466e-15\n", + " NaN\n", + " \n", + " \n", + " std\n", + " NaN\n", + " NaN\n", + " 5.299557e-16\n", + " 5.265197e-16\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " rmse\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 4.077330e-16\n", + " \n", + " \n", + " corr\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", " LHFLX\n", " min\n", " 0.000000e+00\n", @@ -228,6 +424,61 @@ " 0.000000e+00\n", " \n", " \n", + " NETCF\n", + " min\n", + " 0.000000e+00\n", + " 4.383899e-16\n", + " 0.000000e+00\n", + " 4.383899e-16\n", + " 1.387474e-15\n", + " NaN\n", + " \n", + " \n", + " max\n", + " 1.343221e-16\n", + " 1.295151e-16\n", + " 1.343221e-16\n", + " 1.295151e-16\n", + " 6.519133e-16\n", + " NaN\n", + " \n", + " \n", + " mean\n", + " 1.456918e-15\n", + " 1.248197e-15\n", + " 1.456918e-15\n", + " 1.248197e-15\n", + " 1.374967e-15\n", + " NaN\n", + " \n", + " \n", + " std\n", + " NaN\n", + " NaN\n", + " 7.334371e-16\n", + " 2.520113e-16\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " rmse\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " corr\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", " NET_FLUX_SRF\n", " min\n", " 0.000000e+00\n", @@ -448,6 +699,171 @@ " 0.000000e+00\n", " \n", " \n", + " SHFLX\n", + " min\n", + " 1.603024e-16\n", + " 1.656095e-16\n", + " 1.603024e-16\n", + " 1.656095e-16\n", + " 8.953210e-16\n", + " NaN\n", + " \n", + " \n", + " max\n", + " 3.738489e-16\n", + " 0.000000e+00\n", + " 3.738489e-16\n", + " 0.000000e+00\n", + " 2.508920e-16\n", + " NaN\n", + " \n", + " \n", + " mean\n", + " 1.323527e-15\n", + " 5.334326e-16\n", + " 1.323527e-15\n", + " 5.334326e-16\n", + " 1.492293e-15\n", + " NaN\n", + " \n", + " \n", + " std\n", + " NaN\n", + " NaN\n", + " 2.004639e-16\n", + " 3.888756e-16\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " rmse\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " corr\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " SST\n", + " min\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " NaN\n", + " \n", + " \n", + " max\n", + " 3.503644e-16\n", + " 3.599884e-16\n", + " 3.503644e-16\n", + " 3.599884e-16\n", + " 0.000000e+00\n", + " NaN\n", + " \n", + " \n", + " mean\n", + " 5.260736e-15\n", + " 6.503600e-15\n", + " 5.260736e-15\n", + " 6.503600e-15\n", + " 5.172344e-15\n", + " NaN\n", + " \n", + " \n", + " std\n", + " NaN\n", + " NaN\n", + " 2.687098e-15\n", + " 3.161237e-15\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " rmse\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " corr\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " SWCF\n", + " min\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 1.658964e-16\n", + " NaN\n", + " \n", + " \n", + " max\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 3.653508e-15\n", + " NaN\n", + " \n", + " \n", + " mean\n", + " 1.311547e-15\n", + " 1.656736e-15\n", + " 1.311547e-15\n", + " 1.656736e-15\n", + " 1.404070e-15\n", + " NaN\n", + " \n", + " \n", + " std\n", + " NaN\n", + " NaN\n", + " 1.689694e-16\n", + " 0.000000e+00\n", + " NaN\n", + " NaN\n", + " \n", + " \n", + " rmse\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", + " corr\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", + " 0.000000e+00\n", + " \n", + " \n", " TREFHT\n", " min\n", " 2.525625e-16\n", @@ -561,115 +977,210 @@ "" ], "text/plain": [ - " test DIFF (%) ref DIFF (%) test_regrid DIFF (%) \\\n", - "var_key metric \n", - "LHFLX min 0.000000e+00 0.000000e+00 0.000000e+00 \n", - " max 0.000000e+00 2.061090e-16 0.000000e+00 \n", - " mean 1.607934e-15 1.118090e-15 1.607934e-15 \n", - " std NaN NaN 5.414602e-16 \n", - " rmse NaN NaN NaN \n", - " corr NaN NaN NaN \n", - "LWCF min 0.000000e+00 0.000000e+00 0.000000e+00 \n", - " max 0.000000e+00 3.300170e-16 0.000000e+00 \n", - " mean 1.894919e-15 1.310068e-15 1.894919e-15 \n", - " std NaN NaN 6.638807e-16 \n", - " rmse NaN NaN NaN \n", - " corr NaN NaN NaN \n", - "NET_FLUX_SRF min 0.000000e+00 0.000000e+00 0.000000e+00 \n", - " max 1.825516e-16 1.706434e-16 1.825516e-16 \n", - " mean 2.535940e-15 1.465413e-14 2.535940e-15 \n", - " std NaN NaN 3.531975e-16 \n", - " rmse NaN NaN NaN \n", - " corr NaN NaN NaN \n", - "PRECT min 0.000000e+00 0.000000e+00 0.000000e+00 \n", - " max 2.054785e-16 3.506280e-16 2.054785e-16 \n", - " mean 1.308796e-15 1.010973e-15 1.308796e-15 \n", - " std NaN NaN 4.107430e-16 \n", - " rmse NaN NaN NaN \n", - " corr NaN NaN NaN \n", - "PSL min 0.000000e+00 1.168177e-16 0.000000e+00 \n", - " max 2.216200e-16 0.000000e+00 2.216200e-16 \n", - " mean 1.124213e-15 1.236728e-15 1.124213e-15 \n", - " std NaN NaN 5.030916e-16 \n", - " rmse NaN NaN NaN \n", - " corr NaN NaN NaN \n", - "RESTOM min 2.226235e-16 0.000000e+00 2.226235e-16 \n", - " max 0.000000e+00 1.620247e-16 0.000000e+00 \n", - " mean 6.224919e-15 1.107688e-13 6.224919e-15 \n", - " std NaN NaN 3.940225e-16 \n", - " rmse NaN NaN NaN \n", - " corr NaN NaN NaN \n", - "TREFHT min 2.525625e-16 1.221719e-16 2.525625e-16 \n", - " max 1.140829e-16 1.191418e-16 1.140829e-16 \n", - " mean 1.443220e-15 1.026647e-15 1.443220e-15 \n", - " std NaN NaN 3.950659e-16 \n", - " rmse NaN NaN NaN \n", - " corr NaN NaN NaN \n", - " min 2.525625e-16 1.221719e-16 2.525625e-16 \n", - " max 1.140829e-16 1.191418e-16 1.140829e-16 \n", - " mean 9.253642e-15 8.350693e-15 9.253642e-15 \n", - " std NaN NaN 4.922302e-15 \n", - " rmse NaN NaN NaN \n", - " corr NaN NaN NaN \n", + " test DIFF (%) ref DIFF (%) test_regrid DIFF (%) \\\n", + "FLNS min 1.317678e-16 0.000000e+00 1.317678e-16 \n", + " max 0.000000e+00 2.058062e-16 0.000000e+00 \n", + " mean 5.322645e-16 1.276503e-15 5.322645e-16 \n", + " std NaN NaN 5.455412e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "FLUT min 1.140418e-16 0.000000e+00 1.140418e-16 \n", + " max 0.000000e+00 1.893757e-16 0.000000e+00 \n", + " mean 1.779953e-15 1.764910e-15 1.779953e-15 \n", + " std NaN NaN 2.315874e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "FSNS min 2.102518e-16 4.252167e-16 2.102518e-16 \n", + " max 0.000000e+00 2.084296e-16 0.000000e+00 \n", + " mean 2.107767e-15 1.718080e-15 2.107767e-15 \n", + " std NaN NaN 7.842697e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "FSNTOA min 1.582252e-16 0.000000e+00 1.582252e-16 \n", + " max 0.000000e+00 1.569442e-16 0.000000e+00 \n", + " mean 8.294511e-16 9.417413e-16 8.294511e-16 \n", + " std NaN NaN 5.299557e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "LHFLX min 0.000000e+00 0.000000e+00 0.000000e+00 \n", + " max 0.000000e+00 2.061090e-16 0.000000e+00 \n", + " mean 1.607934e-15 1.118090e-15 1.607934e-15 \n", + " std NaN NaN 5.414602e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "LWCF min 0.000000e+00 0.000000e+00 0.000000e+00 \n", + " max 0.000000e+00 3.300170e-16 0.000000e+00 \n", + " mean 1.894919e-15 1.310068e-15 1.894919e-15 \n", + " std NaN NaN 6.638807e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "NETCF min 0.000000e+00 4.383899e-16 0.000000e+00 \n", + " max 1.343221e-16 1.295151e-16 1.343221e-16 \n", + " mean 1.456918e-15 1.248197e-15 1.456918e-15 \n", + " std NaN NaN 7.334371e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "NET_FLUX_SRF min 0.000000e+00 0.000000e+00 0.000000e+00 \n", + " max 1.825516e-16 1.706434e-16 1.825516e-16 \n", + " mean 2.535940e-15 1.465413e-14 2.535940e-15 \n", + " std NaN NaN 3.531975e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "PRECT min 0.000000e+00 0.000000e+00 0.000000e+00 \n", + " max 2.054785e-16 3.506280e-16 2.054785e-16 \n", + " mean 1.308796e-15 1.010973e-15 1.308796e-15 \n", + " std NaN NaN 4.107430e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "PSL min 0.000000e+00 1.168177e-16 0.000000e+00 \n", + " max 2.216200e-16 0.000000e+00 2.216200e-16 \n", + " mean 1.124213e-15 1.236728e-15 1.124213e-15 \n", + " std NaN NaN 5.030916e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "RESTOM min 2.226235e-16 0.000000e+00 2.226235e-16 \n", + " max 0.000000e+00 1.620247e-16 0.000000e+00 \n", + " mean 6.224919e-15 1.107688e-13 6.224919e-15 \n", + " std NaN NaN 3.940225e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "SHFLX min 1.603024e-16 1.656095e-16 1.603024e-16 \n", + " max 3.738489e-16 0.000000e+00 3.738489e-16 \n", + " mean 1.323527e-15 5.334326e-16 1.323527e-15 \n", + " std NaN NaN 2.004639e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "SST min 0.000000e+00 0.000000e+00 0.000000e+00 \n", + " max 3.503644e-16 3.599884e-16 3.503644e-16 \n", + " mean 5.260736e-15 6.503600e-15 5.260736e-15 \n", + " std NaN NaN 2.687098e-15 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "SWCF min 0.000000e+00 0.000000e+00 0.000000e+00 \n", + " max 0.000000e+00 0.000000e+00 0.000000e+00 \n", + " mean 1.311547e-15 1.656736e-15 1.311547e-15 \n", + " std NaN NaN 1.689694e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + "TREFHT min 2.525625e-16 1.221719e-16 2.525625e-16 \n", + " max 1.140829e-16 1.191418e-16 1.140829e-16 \n", + " mean 1.443220e-15 1.026647e-15 1.443220e-15 \n", + " std NaN NaN 3.950659e-16 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", + " min 2.525625e-16 1.221719e-16 2.525625e-16 \n", + " max 1.140829e-16 1.191418e-16 1.140829e-16 \n", + " mean 9.253642e-15 8.350693e-15 9.253642e-15 \n", + " std NaN NaN 4.922302e-15 \n", + " rmse NaN NaN NaN \n", + " corr NaN NaN NaN \n", "\n", - " ref_regrid DIFF (%) diff DIFF (%) misc DIFF (%) \n", - "var_key metric \n", - "LHFLX min 0.000000e+00 2.066978e-16 NaN \n", - " max 2.061090e-16 0.000000e+00 NaN \n", - " mean 1.118090e-15 1.693728e-15 NaN \n", - " std 5.367577e-16 NaN NaN \n", - " rmse NaN NaN 0.000000e+00 \n", - " corr NaN NaN 0.000000e+00 \n", - "LWCF min 0.000000e+00 1.343861e-15 NaN \n", - " max 3.300170e-16 0.000000e+00 NaN \n", - " mean 1.310068e-15 2.985093e-14 NaN \n", - " std 1.639685e-16 NaN NaN \n", - " rmse NaN NaN 0.000000e+00 \n", - " corr NaN NaN 0.000000e+00 \n", - "NET_FLUX_SRF min 0.000000e+00 1.873360e-16 NaN \n", - " max 1.706434e-16 0.000000e+00 NaN \n", - " mean 1.465413e-14 2.161829e-15 NaN \n", - " std 4.735973e-16 NaN NaN \n", - " rmse NaN NaN 0.000000e+00 \n", - " corr NaN NaN 0.000000e+00 \n", - "PRECT min 0.000000e+00 0.000000e+00 NaN \n", - " max 3.506280e-16 5.683466e-16 NaN \n", - " mean 1.010973e-15 0.000000e+00 NaN \n", - " std 4.058335e-16 NaN NaN \n", - " rmse NaN NaN 0.000000e+00 \n", - " corr NaN NaN 0.000000e+00 \n", - "PSL min 1.168177e-16 3.592752e-14 NaN \n", - " max 0.000000e+00 3.973074e-14 NaN \n", - " mean 1.236728e-15 1.300318e-14 NaN \n", - " std 3.531204e-16 NaN NaN \n", - " rmse NaN NaN 1.064570e-15 \n", - " corr NaN NaN 0.000000e+00 \n", - "RESTOM min 0.000000e+00 4.666565e-16 NaN \n", - " max 1.620247e-16 8.168903e-16 NaN \n", - " mean 1.107688e-13 2.155737e-15 NaN \n", - " std 3.955053e-16 NaN NaN \n", - " rmse NaN NaN 2.985625e-16 \n", - " corr NaN NaN 0.000000e+00 \n", - "TREFHT min 1.221719e-16 1.466053e-15 NaN \n", - " max 1.191418e-16 3.565723e-16 NaN \n", - " mean 1.026647e-15 1.076803e-15 NaN \n", - " std 5.141888e-16 NaN NaN \n", - " rmse NaN NaN 0.000000e+00 \n", - " corr NaN NaN 0.000000e+00 \n", - " min 1.221719e-16 1.466053e-15 NaN \n", - " max 1.191418e-16 7.298314e-16 NaN \n", - " mean 8.350693e-15 9.621027e-15 NaN \n", - " std 4.475437e-15 NaN NaN \n", - " rmse NaN NaN 0.000000e+00 \n", - " corr NaN NaN 0.000000e+00 " + " ref_regrid DIFF (%) diff DIFF (%) misc DIFF (%) \n", + "FLNS min 0.000000e+00 0.000000e+00 NaN \n", + " max 2.058062e-16 7.203322e-16 NaN \n", + " mean 1.276503e-15 1.764203e-15 NaN \n", + " std 5.344197e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "FLUT min 0.000000e+00 0.000000e+00 NaN \n", + " max 1.893757e-16 0.000000e+00 NaN \n", + " mean 1.764910e-15 1.522792e-15 NaN \n", + " std 4.524283e-16 NaN NaN \n", + " rmse NaN NaN 2.566283e-16 \n", + " corr NaN NaN 0.000000e+00 \n", + "FSNS min 4.252167e-16 9.861022e-16 NaN \n", + " max 2.084296e-16 3.441230e-16 NaN \n", + " mean 1.718080e-15 1.350994e-15 NaN \n", + " std 2.191950e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "FSNTOA min 0.000000e+00 1.205526e-15 NaN \n", + " max 1.569442e-16 1.527862e-15 NaN \n", + " mean 9.417413e-16 1.405466e-15 NaN \n", + " std 5.265197e-16 NaN NaN \n", + " rmse NaN NaN 4.077330e-16 \n", + " corr NaN NaN 0.000000e+00 \n", + "LHFLX min 0.000000e+00 2.066978e-16 NaN \n", + " max 2.061090e-16 0.000000e+00 NaN \n", + " mean 1.118090e-15 1.693728e-15 NaN \n", + " std 5.367577e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "LWCF min 0.000000e+00 1.343861e-15 NaN \n", + " max 3.300170e-16 0.000000e+00 NaN \n", + " mean 1.310068e-15 2.985093e-14 NaN \n", + " std 1.639685e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "NETCF min 4.383899e-16 1.387474e-15 NaN \n", + " max 1.295151e-16 6.519133e-16 NaN \n", + " mean 1.248197e-15 1.374967e-15 NaN \n", + " std 2.520113e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "NET_FLUX_SRF min 0.000000e+00 1.873360e-16 NaN \n", + " max 1.706434e-16 0.000000e+00 NaN \n", + " mean 1.465413e-14 2.161829e-15 NaN \n", + " std 4.735973e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "PRECT min 0.000000e+00 0.000000e+00 NaN \n", + " max 3.506280e-16 5.683466e-16 NaN \n", + " mean 1.010973e-15 0.000000e+00 NaN \n", + " std 4.058335e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "PSL min 1.168177e-16 3.592752e-14 NaN \n", + " max 0.000000e+00 3.973074e-14 NaN \n", + " mean 1.236728e-15 1.300318e-14 NaN \n", + " std 3.531204e-16 NaN NaN \n", + " rmse NaN NaN 1.064570e-15 \n", + " corr NaN NaN 0.000000e+00 \n", + "RESTOM min 0.000000e+00 4.666565e-16 NaN \n", + " max 1.620247e-16 8.168903e-16 NaN \n", + " mean 1.107688e-13 2.155737e-15 NaN \n", + " std 3.955053e-16 NaN NaN \n", + " rmse NaN NaN 2.985625e-16 \n", + " corr NaN NaN 0.000000e+00 \n", + "SHFLX min 1.656095e-16 8.953210e-16 NaN \n", + " max 0.000000e+00 2.508920e-16 NaN \n", + " mean 5.334326e-16 1.492293e-15 NaN \n", + " std 3.888756e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "SST min 0.000000e+00 0.000000e+00 NaN \n", + " max 3.599884e-16 0.000000e+00 NaN \n", + " mean 6.503600e-15 5.172344e-15 NaN \n", + " std 3.161237e-15 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "SWCF min 0.000000e+00 1.658964e-16 NaN \n", + " max 0.000000e+00 3.653508e-15 NaN \n", + " mean 1.656736e-15 1.404070e-15 NaN \n", + " std 0.000000e+00 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + "TREFHT min 1.221719e-16 1.466053e-15 NaN \n", + " max 1.191418e-16 3.565723e-16 NaN \n", + " mean 1.026647e-15 1.076803e-15 NaN \n", + " std 5.141888e-16 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 \n", + " min 1.221719e-16 1.466053e-15 NaN \n", + " max 1.191418e-16 7.298314e-16 NaN \n", + " mean 8.350693e-15 9.621027e-15 NaN \n", + " std 4.475437e-15 NaN NaN \n", + " rmse NaN NaN 0.000000e+00 \n", + " corr NaN NaN 0.000000e+00 " ] }, - "execution_count": 51, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "pd.set_option(\"display.max_rows\", None)\n", "df_metrics_diffs" ] }, @@ -677,21 +1188,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Filter differences to those above maximum threshold (2%).\n", - "\n", - "All values below maximum threshold will be labeled as `NaN`.\n", - "\n", - "- **If all cells in a row are NaN (< 2%)**, the entire row is dropped to make the results easier to parse.\n", - "- Any remaining NaN cells are below < 2% difference and **should be ignored**.\n" + "## 2. Filter differences to those above maximum threshold (1e-10 | 1e-8%).\n" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "df_metrics_diffs_thres = df_metrics_diffs[df_metrics_diffs >= 0.02]\n", + "df_metrics_diffs_thres = df_metrics_diffs[df_metrics_diffs >= 1e-10]\n", "df_metrics_diffs_thres = df_metrics_diffs_thres.dropna(\n", " axis=0, how=\"all\", ignore_index=False\n", ")" @@ -699,7 +1205,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -731,16 +1237,6 @@ " diff DIFF (%)\n", " misc DIFF (%)\n", " \n", - " \n", - " var_key\n", - " metric\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", " \n", @@ -753,7 +1249,7 @@ "Index: []" ] }, - "execution_count": 57, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -766,14 +1262,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Findings: No metrics are above the 2% threshold after removing the `slice_flag` used in\n", - "the CDAT version of the codebase.\n" + "Findings: No metrics are above the 1e-10 (1e-8%) threshold after removing the `slice_flag` used in\n", + "the CDAT version of the codebase. Results are virtually identical now.\n" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] } ], "metadata": {