Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[python] reset storage in record evaluation callback each time before starting training #4885

Merged
merged 5 commits into from
Dec 18, 2021
Merged

[python] reset storage in record evaluation callback each time before starting training #4885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ea042fd
Update test_sklearn.py
StrikerRUS Dec 12, 2021
085d18b
Update python_package.yml
StrikerRUS Dec 12, 2021
08c8c4e
Update python_package.yml
StrikerRUS Dec 12, 2021
36208ca
Update callback.py
StrikerRUS Dec 12, 2021
1ea4585
Update callback.py
StrikerRUS Dec 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 3 additions & 10 deletions python-package/lightgbm/callback.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,15 +128,15 @@ def record_evaluation(eval_result: Dict[str, Dict[str, List[Any]]]) -> Callable:
"""
if not isinstance(eval_result, dict):
raise TypeError('eval_result should be a dictionary')
eval_result.clear()

def _init(env: CallbackEnv) -> None:
eval_result.clear()
for data_name, eval_name, _, _ in env.evaluation_result_list:
eval_result.setdefault(data_name, collections.OrderedDict())
eval_result[data_name].setdefault(eval_name, [])

def _callback(env: CallbackEnv) -> None:
if not eval_result:
if env.iteration == env.begin_iteration:
_init(env)
for data_name, eval_name, result, _ in env.evaluation_result_list:
eval_result[data_name][eval_name].append(result)
Expand Down Expand Up @@ -221,7 +221,6 @@ def early_stopping(stopping_rounds: int, first_metric_only: bool = False, verbos
best_score_list: list = []
cmp_op = []
enabled = True
inited = False
first_metric = ''

def _init(env: CallbackEnv) -> None:
Expand All @@ -230,7 +229,6 @@ def _init(env: CallbackEnv) -> None:
nonlocal best_score_list
nonlocal cmp_op
nonlocal enabled
nonlocal inited
nonlocal first_metric
enabled = not any(env.params.get(boost_alias, "") == 'dart' for boost_alias
in _ConfigAliases.get("boosting"))
Expand All @@ -249,7 +247,6 @@ def _init(env: CallbackEnv) -> None:
best_iter = []
best_score_list = []
cmp_op = []
inited = True
first_metric = ''

n_metrics = len(set(m[1] for m in env.evaluation_result_list))
Expand Down Expand Up @@ -293,15 +290,13 @@ def _init(env: CallbackEnv) -> None:
def _final_iteration_check(env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None:
nonlocal best_iter
nonlocal best_score_list
nonlocal inited
if env.iteration == env.end_iteration - 1:
if verbose:
best_score_str = '\t'.join([_format_eval_result(x) for x in best_score_list[i]])
_log_info('Did not meet early stopping. '
f'Best iteration is:\n[{best_iter[i] + 1}]\t{best_score_str}')
if first_metric_only:
_log_info(f"Evaluated only: {eval_name_splitted[-1]}")
inited = False
raise EarlyStopException(best_iter[i], best_score_list[i])

def _callback(env: CallbackEnv) -> None:
Expand All @@ -310,9 +305,8 @@ def _callback(env: CallbackEnv) -> None:
nonlocal best_score_list
nonlocal cmp_op
nonlocal enabled
nonlocal inited
nonlocal first_metric
if not inited:
if env.iteration == env.begin_iteration:
_init(env)
if not enabled:
return
Expand All @@ -336,7 +330,6 @@ def _callback(env: CallbackEnv) -> None:
_log_info(f"Early stopping, best iteration is:\n[{best_iter[i] + 1}]\t{eval_result_str}")
if first_metric_only:
_log_info(f"Evaluated only: {eval_name_splitted[-1]}")
inited = False
raise EarlyStopException(best_iter[i], best_score_list[i])
_final_iteration_check(env, eval_name_splitted, i)
_callback.order = 30 # type: ignore
Expand Down
22 changes: 13 additions & 9 deletions tests/python_package_test/test_sklearn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,20 +294,23 @@ def test_stacking_regressor():
def test_grid_search():
X, y = load_iris(return_X_y=True)
y = y.astype(str) # utilize label encoder at it's max power
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1,
random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1,
random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=42)
params = dict(subsample=0.8,
subsample_freq=1)
grid_params = dict(boosting_type=['rf', 'gbdt'],
n_estimators=[4, 6],
reg_alpha=[0.01, 0.005])
fit_params = dict(eval_set=[(X_val, y_val)],
eval_metric=constant_metric,
callbacks=[lgb.early_stopping(2)])
grid = GridSearchCV(estimator=lgb.LGBMClassifier(**params), param_grid=grid_params,
cv=2)
evals_result = {}
fit_params = dict(
eval_set=[(X_val, y_val)],
eval_metric=constant_metric,
callbacks=[
lgb.early_stopping(2),
lgb.record_evaluation(evals_result)
]
)
grid = GridSearchCV(estimator=lgb.LGBMClassifier(**params), param_grid=grid_params, cv=2)
grid.fit(X_train, y_train, **fit_params)
score = grid.score(X_test, y_test) # utilizes GridSearchCV default refit=True
assert grid.best_params_['boosting_type'] in ['rf', 'gbdt']
Expand All @@ -319,6 +322,7 @@ def test_grid_search():
assert grid.best_estimator_.best_score_['valid_0']['error'] == 0
assert score >= 0.2
assert score <= 1.
assert evals_result == grid.best_estimator_.evals_result_
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This assert fails at master:

>       assert evals_result == grid.best_estimator_.evals_result_
E       AssertionError: assert {'valid_0': O..., 0, 0, 0])])} == {'valid_0': O... [0, 0, 0])])}
E         Differing items:
E         {'valid_0': OrderedDict([('multi_logloss', [0.5915701709051111, 0.5814219901209461, 0.6047065388140261, 0.555159764382..., 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])])} != {'valid_0': OrderedDict([('multi_logloss', [0.11975431959688579, 0.11424123104047454, 0.11872456319439122]), ('error', [0, 0, 0])])}

https://github.com/microsoft/LightGBM/runs/4499752025?check_suite_focus=true#step:3:834



def test_random_search():
Expand Down