Upgrade AzureML cpu cluster to a higher tier

[miguelgfierro]

Description

Using VMs with 64GB of RAM. Updated the AzureML test workspace with Standard_A8m_v2 (8 cores, 64 GB RAM, 80 GB disk)

Related Issues

References

Checklist:

• I have followed the contribution guidelines and code style for this project.
• I have added tests covering my contributions.
• I have updated the documentation accordingly.
• This PR is being made to staging branch and not to main branch.

[miguelgfierro]

Running the CPU nightly builds in the new cluster: https://github.com/microsoft/recommenders/actions/runs/4305702066/jobs/7508521965

[miguelgfierro]

Error in the performance test because we are using a slower VM:

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py F [ 81%]
FFFFFFFFFFFFF                                                            [100%]

=================================== FAILURES ===================================
______________________________ test_merge_rating _______________________________

rating_true =          userID  itemID  rating
0          5853   13436       1
1          5050   11360       1
2          1120   1146...97     772    4731       3
9999998    7071   15052       2
9999999     695   15212       5

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          1357   16636           5
1           205   16049           3
2         ... 13396           3
9999998    9277    5263           2
9999999    3843   14429           4

[10000000 rows x 3 columns]

    def test_merge_rating(rating_true, rating_pred):
        with Timer() as t:
            merge_rating_true_pred(
                rating_true,
                rating_pred,
                col_user=DEFAULT_USER_COL,
                col_item=DEFAULT_ITEM_COL,
                col_rating=DEFAULT_RATING_COL,
                col_prediction=DEFAULT_PREDICTION_COL,
            )
>       assert t.interval < 19.81252386 * (1 + TOL)
E       assert 43.44402541999989 < (19.81252386 * (1 + 0.1))
E        +  where 43.44402541999989 = <recommenders.utils.timer.Timer object at 0x7f759df28610>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:101: AssertionError
______________________________ test_merge_ranking ______________________________

rating_true =          userID  itemID  rating
0          3864    7896       4
1          5699    1224       2
2          1283    725...97    1218   11638       3
9999998    1506    1692       2
9999999    6593    7543       1

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          7002   16561           3
1          9511    1631           5
2         ...  9045           4
9999998    2305    2494           1
9999999    6297   12292           3

[10000000 rows x 3 columns]

    def test_merge_ranking(rating_true, rating_pred):
        with Timer() as t:
            merge_ranking_true_pred(
                rating_true,
                rating_pred,
                col_user=DEFAULT_USER_COL,
                col_item=DEFAULT_ITEM_COL,
                col_rating=DEFAULT_RATING_COL,
                col_prediction=DEFAULT_PREDICTION_COL,
                relevancy_method="top_k",
            )
>       assert t.interval < 22.01327171 * (1 + TOL)
E       assert 48.376839234000045 < (22.01327171 * (1 + 0.1))
E        +  where 48.376839234000045 = <recommenders.utils.timer.Timer object at 0x7f75a4b294f0>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:115: AssertionError
_______________________________ test_python_rmse _______________________________

rating_true =          userID  itemID  rating
0          7171   15715       4
1          4892   15076       3
2          5992   1878...97    8605   11308       4
9999998    6767   19971       5
9999999    7218    9462       5

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          3917   14815           2
1          2091    2121           2
2         ...  5920           4
9999998    8919    5661           4
9999999    1453    3235           1

[10000000 rows x 3 columns]

    def test_python_rmse(rating_true, rating_pred):
        with Timer() as t:
            rmse(
                rating_true=rating_true,
                rating_pred=rating_pred,
                col_prediction=DEFAULT_PREDICTION_COL,
            )
>       assert t.interval < 29.95031411 * (1 + TOL)
E       assert 43.099514309999904 < (29.95031411 * (1 + 0.1))
E        +  where 43.099514309999904 = <recommenders.utils.timer.Timer object at 0x7f759df28c10>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:125: AssertionError
_______________________________ test_python_mae ________________________________

rating_true =          userID  itemID  rating
0          2145   15612       2
1          1984   16757       4
2          1286   1526...97     220   10769       2
9999998    7872   10269       1
9999999    6666   16751       4

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          9608   18996           3
1          7041    5874           4
2         ... 14035           2
9999998    2744   12215           3
9999999    6167   16720           1

[10000000 rows x 3 columns]

    def test_python_mae(rating_true, rating_pred):
        with Timer() as t:
            mae(
                rating_true=rating_true,
                rating_pred=rating_pred,
                col_prediction=DEFAULT_PREDICTION_COL,
            )
>       assert t.interval < 30.45756622 * (1 + TOL)
E       assert 43.444752608000044 < (30.45756622 * (1 + 0.1))
E        +  where 43.444752608000044 = <recommenders.utils.timer.Timer object at 0x7f759df1fb20>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:135: AssertionError
_____________________________ test_python_rsquared _____________________________

rating_true =          userID  itemID  rating
0          5178    6519       5
1          9344    6533       2
2           640   1747...97    1539    6428       5
9999998    8311     367       2
9999999    2229   10300       5

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          7262   13663           3
1          2979    1454           5
2         ... 17264           5
9999998    8659   10877           5
9999999    3377   13874           5

[10000000 rows x 3 columns]

    def test_python_rsquared(rating_true, rating_pred):
        with Timer() as t:
            rsquared(
                rating_true=rating_true,
                rating_pred=rating_pred,
                col_prediction=DEFAULT_PREDICTION_COL,
            )
>       assert t.interval < 30.60572284 * (1 + TOL)
E       assert 43.578797186999964 < (30.60572284 * (1 + 0.1))
E        +  where 43.578797186999964 = <recommenders.utils.timer.Timer object at 0x7f759ee9e6a0>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:145: AssertionError
_____________________________ test_python_exp_var ______________________________

rating_true =          userID  itemID  rating
0          2362    1269       5
1          1343     430       3
2          3895    432...97    4984    8622       4
9999998    2259   13807       2
9999999    8599    9180       2

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          4236   15732           3
1          7123   16561           3
2         ...  4296           4
9999998    1891    1753           3
9999999    7785   10307           4

[10000000 rows x 3 columns]

    def test_python_exp_var(rating_true, rating_pred):
        with Timer() as t:
            exp_var(
                rating_true=rating_true,
                rating_pred=rating_pred,
                col_prediction=DEFAULT_PREDICTION_COL,
            )
>       assert t.interval < 31.01451915 * (1 + TOL)
E       assert 44.0[936](https://github.com/microsoft/recommenders/actions/runs/4313836357/jobs/7526133902#step:3:945)04943999935 < (31.01451915 * (1 + 0.1))
E        +  where 44.093604943999935 = <recommenders.utils.timer.Timer object at 0x7f759eea1250>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:155: AssertionError
_____________________________ test_get_top_k_items _____________________________

rating_true =          userID  itemID  rating
0          8351   15342       3
1          1220   11095       3
2          2759    453...97    8155    6908       5
9999998    7084    7818       4
9999999    3878   14106       4

[10000000 rows x 3 columns]

    def test_get_top_k_items(rating_true):
        with Timer() as t:
            get_top_k_items(
                dataframe=rating_true,
                col_user=DEFAULT_USER_COL,
                col_rating=DEFAULT_RATING_COL,
                k=10,
            )
>       assert t.interval < 3.90527593 * (1 + TOL)
E       assert 8.567175116000044 < (3.90527593 * (1 + 0.1))
E        +  where 8.567175116000044 = <recommenders.utils.timer.Timer object at 0x7f759eea1af0>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:166: AssertionError
_________________________ test_get_top_k_items_largek __________________________

rating_true =          userID  itemID  rating
0          4590   18393       4
1          7431    3189       5
2          4144     71...97    2749   16177       4
9999998    4228   17245       3
9999999    1931   12192       2

[10000000 rows x 3 columns]

    def test_get_top_k_items_largek(rating_true):
        with Timer() as t:
            get_top_k_items(
                dataframe=rating_true,
                col_user=DEFAULT_USER_COL,
                col_rating=DEFAULT_RATING_COL,
                k=1000,
            )
>       assert t.interval < 4.83998316 * (1 + TOL)
E       assert 10.599288784999999 < (4.83998316 * (1 + 0.1))
E        +  where 10.599288784999999 = <recommenders.utils.timer.Timer object at 0x7f759df0ffa0>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:177: AssertionError
____________________________ test_python_ndcg_at_k _____________________________

rating_true =          userID  itemID  rating
0          5561   14622       5
1          7768   13517       5
2          5118   1232...97    6823    2544       4
9999998    1993   13444       2
9999999    2268    5734       5

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          6440   16612           3
1          2099    7644           2
2         ... 12157           2
9999998    4083   19101           5
9999999    7258    8568           1

[10000000 rows x 3 columns]

    def test_python_ndcg_at_k(rating_true, rating_pred):
        with Timer() as t:
            ndcg_at_k(
                rating_true=rating_true,
                rating_pred=rating_pred,
                col_prediction=DEFAULT_PREDICTION_COL,
                k=10,
            )
>       assert t.interval < 39.03877957 * (1 + TOL)
E       assert 89.16398501499998 < (39.03877957 * (1 + 0.1))
E        +  where 89.16398501499998 = <recommenders.utils.timer.Timer object at 0x7f75a58b2670>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:188: AssertionError
_____________________________ test_python_map_at_k _____________________________

rating_true =          userID  itemID  rating
0          5997    4762       2
1          3023   11912       1
2            42    149...97    3319   10879       3
9999998    8775   12381       2
9999999    4867   18081       5

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          1393    9477           2
1          4377   19397           3
2         ... 13095           1
9999998    3557    9354           4
9999999    2434   10863           1

[10000000 rows x 3 columns]

    def test_python_map_at_k(rating_true, rating_pred):
        with Timer() as t:
            map_at_k(
                rating_true=rating_true,
                rating_pred=rating_pred,
                col_prediction=DEFAULT_PREDICTION_COL,
                k=10,
            )
>       assert t.interval < 29.90376154 * (1 + TOL)
E       assert 48.91643127799989 < (29.90376154 * (1 + 0.1))
E        +  where 48.91643127799989 = <recommenders.utils.timer.Timer object at 0x7f75a58b5b50>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:199: AssertionError
____________________________ test_python_precision _____________________________

rating_true =          userID  itemID  rating
0          5830    8893       3
1          1539   11461       4
2          2957    943...97    5190   10147       1
9999998    7955      84       5
9999999    6805    2886       3

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          2884   14815           3
1          2175   18382           3
2         ... 10225           4
9999998    3763    7163           5
9999999    6705   18927           3

[10000000 rows x 3 columns]

    def test_python_precision(rating_true, rating_pred):
        with Timer() as t:
            precision_at_k(rating_true, rating_pred, k=10)
>       assert t.interval < 29.95129834 * (1 + TOL)
E       assert 49.27300530399998 < (29.95129834 * (1 + 0.1))
E        +  where 49.27300530399998 = <recommenders.utils.timer.Timer object at 0x7f759eeaf9d0>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:205: AssertionError
______________________________ test_python_recall ______________________________

rating_true =          userID  itemID  rating
0          7682   12955       4
1          4636    6651       3
2           700   1816...97    9384   19248       3
9999998    2504   10984       4
9999999    6618   16372       2

[10000000 rows x 3 columns]
rating_pred =          userID  itemID  prediction
0          9354   17234           4
1          5322    1074           2
2         ... 14775           1
9999998    4324   11446           5
9999999    7247   12123           2

[10000000 rows x 3 columns]

    def test_python_recall(rating_true, rating_pred):
        with Timer() as t:
            recall_at_k(
                rating_true=rating_true,
                rating_pred=rating_pred,
                col_prediction=DEFAULT_PREDICTION_COL,
                k=10,
            )
>       assert t.interval < 30.29558967 * (1 + TOL)
E       assert 49.400000267999985 < (30.29558967 * (1 + 0.1))
E        +  where 49.400000267999985 = <recommenders.utils.timer.Timer object at 0x7f759eea8af0>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:216: AssertionError
_______________________________ test_python_auc ________________________________

rating_true_binary =          userID  itemID  rating
0          5498   19230     0.0
1          4979   13733     0.0
2          7233    807...97    7972    5747     0.0
9999998    7775    6614     0.0
9999999    8187   15484     0.0

[10000000 rows x 3 columns]
rating_pred_binary =          userID  itemID  prediction
0          8417   12729        1.00
1          8649    5663        0.75
2         ...  2573        1.00
9999998    4993    8675        0.50
9999999     123    9681        0.50

[10000000 rows x 3 columns]

    def test_python_auc(rating_true_binary, rating_pred_binary):
        with Timer() as t:
            auc(
                rating_true=rating_true_binary,
                rating_pred=rating_pred_binary,
                col_rating=DEFAULT_RATING_COL,
                col_prediction=DEFAULT_PREDICTION_COL,
            )
>       assert t.interval < 22.18870257 * (1 + TOL)
E       assert 44.20187142899999 < (22.18870257 * (1 + 0.1))
E        +  where 44.20187142899999 = <recommenders.utils.timer.Timer object at 0x7f759eeb21f0>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:227: AssertionError
_____________________________ test_python_logloss ______________________________

rating_true_binary =          userID  itemID  rating
0          8089    2418     1.0
1          4658   18942     1.0
2          9162    [937](https://github.com/microsoft/recommenders/actions/runs/4313836357/jobs/7526133902#step:3:946)...97    9675    8077     0.0
9999998    [961](https://github.com/microsoft/recommenders/actions/runs/4313836357/jobs/7526133902#step:3:970)8   10403     0.0
9999999    5242   11564     1.0

[10000000 rows x 3 columns]
rating_pred_binary =          userID  itemID  prediction
0          4918   15795        1.00
1          8652    2877        1.00
2         ... 13865        0.75
9999998    4106    8110        0.50
9999999    3770    8446        0.75

[10000000 rows x 3 columns]

    def test_python_logloss(rating_true_binary, rating_pred_binary):
        with Timer() as t:
            logloss(
                rating_true=rating_true_binary,
                rating_pred=rating_pred_binary,
                col_rating=DEFAULT_RATING_COL,
                col_prediction=DEFAULT_PREDICTION_COL,
            )
>       assert t.interval < 32.9162[978](https://github.com/microsoft/recommenders/actions/runs/4313836357/jobs/7526133902#step:3:987)7 * (1 + TOL)
E       assert 43.78255590899[994](https://github.com/microsoft/recommenders/actions/runs/4313836357/jobs/7526133902#step:3:1003) < (32.91629787 * (1 + 0.1))
E        +  where 43.782555908[999](https://github.com/microsoft/recommenders/actions/runs/4313836357/jobs/7526133902#step:3:1008)94 = <recommenders.utils.timer.Timer object at 0x7f759eea7760>.interval

tests/unit/recommenders/evaluation/test_python_evaluation_time_performance.py:238: AssertionError

[miguelgfierro]

This error won't happen if we had #1774