Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[AutoTVM] Fix hang/crash issues on feature extraction #3689

Merged
merged 3 commits into from
Aug 2, 2019
Merged

[AutoTVM] Fix hang/crash issues on feature extraction #3689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
09c820a
[AutoTVM] Fix hang/crash issues on feature extraction
merrymercy Aug 1, 2019
5c7f0a5
Update xgboost_cost_model.py
merrymercy Aug 2, 2019
542f985
fix lint
merrymercy Aug 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 76 additions & 52 deletions python/tvm/autotvm/tuner/xgboost_cost_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,16 @@ def _get_feature(self, indexes):
for i, fea in zip(need_extract, feas):
fea_cache[i] = fea

ret = np.empty((len(indexes), fea_cache[indexes[0]].shape[-1]), dtype=np.float32)
feature_len = None
for idx in indexes:
if fea_cache[idx] is not None:
feature_len = fea_cache[idx].shape[-1]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is fea_cache[idx].shape[-1] the same for all non-None element in fea_cache?
Is break missing here?

Copy link
Member Author

@merrymercy merrymercy Aug 2, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. A break is missing.
All non-None elements should have the same length, otherwise it will raise error in L324

break

ret = np.empty((len(indexes), feature_len), dtype=np.float32)
for i, ii in enumerate(indexes):
ret[i, :] = fea_cache[ii]
t = fea_cache[ii]
ret[i, :] = t if t is not None else 0
return ret

def __del__(self):
Expand All @@ -327,71 +334,88 @@ def __del__(self):

def _extract_itervar_feature_index(index):
"""extract iteration var feature for an index in extract_space"""
config = _extract_space.get(index)
with _extract_target:
sch, args = _extract_task.instantiate(config)
fea = feature.get_itervar_feature_flatten(sch, args, take_log=True)
fea = np.concatenate((fea, list(config.get_other_option().values())))
return fea
try:
config = _extract_space.get(index)
with _extract_target:
sch, args = _extract_task.instantiate(config)
fea = feature.get_itervar_feature_flatten(sch, args, take_log=True)
fea = np.concatenate((fea, list(config.get_other_option().values())))
return fea
except Exception: # pylint: disable=broad-except
return None

def _extract_itervar_feature_log(arg):
"""extract iteration var feature for log items"""
inp, res = arg
config = inp.config
with inp.target:
sch, args = inp.task.instantiate(config)
fea = feature.get_itervar_feature_flatten(sch, args, take_log=True)
x = np.concatenate((fea, list(config.get_other_option().values())))

if res.error_no == 0:
y = inp.task.flop / np.mean(res.costs)
else:
y = 0.0
return x, y
try:
inp, res = arg
config = inp.config
with inp.target:
sch, args = inp.task.instantiate(config)
fea = feature.get_itervar_feature_flatten(sch, args, take_log=True)
x = np.concatenate((fea, list(config.get_other_option().values())))

if res.error_no == 0:
y = inp.task.flop / np.mean(res.costs)
else:
y = 0.0
return x, y
except Exception: # pylint: disable=broad-except
return None

def _extract_knob_feature_index(index):
"""extract knob feature for an index in extract_space"""
config = _extract_space.get(index)
return config.get_flatten_feature()
try:
config = _extract_space.get(index)
return config.get_flatten_feature()
except Exception: # pylint: disable=broad-except
return None

def _extract_knob_feature_log(arg):
"""extract knob feature for log items"""
inp, res = arg
config = inp.config
x = config.get_flatten_feature()

if res.error_no == 0:
with inp.target: # necessary, for calculating flops of this task
inp.task.instantiate(config)
y = inp.task.flop / np.mean(res.costs)
else:
y = 0.0
return x, y
try:
inp, res = arg
config = inp.config
x = config.get_flatten_feature()

if res.error_no == 0:
with inp.target: # necessary, for calculating flops of this task
inp.task.instantiate(config)
y = inp.task.flop / np.mean(res.costs)
else:
y = 0.0
return x, y
except Exception: # pylint: disable=broad-except
return None

def _extract_curve_feature_index(index):
"""extract sampled curve feature for an index in extract_space"""
config = _extract_space.get(index)
with _extract_target:
sch, args = _extract_task.instantiate(config)
fea = feature.get_buffer_curve_sample_flatten(sch, args, sample_n=20)
fea = np.concatenate((fea, list(config.get_other_option().values())))
return np.array(fea)
try:
config = _extract_space.get(index)
with _extract_target:
sch, args = _extract_task.instantiate(config)
fea = feature.get_buffer_curve_sample_flatten(sch, args, sample_n=20)
fea = np.concatenate((fea, list(config.get_other_option().values())))
return np.array(fea)
except Exception: # pylint: disable=broad-except
return None

def _extract_curve_feature_log(arg):
"""extract sampled curve feature for log items"""
inp, res = arg
config = inp.config
with inp.target:
sch, args = inp.task.instantiate(config)
fea = feature.get_buffer_curve_sample_flatten(sch, args, sample_n=20)
x = np.concatenate((fea, list(config.get_other_option().values())))

if res.error_no == 0:
y = inp.task.flop / np.mean(res.costs)
else:
y = 0.0
return x, y

try:
inp, res = arg
config = inp.config
with inp.target:
sch, args = inp.task.instantiate(config)
fea = feature.get_buffer_curve_sample_flatten(sch, args, sample_n=20)
x = np.concatenate((fea, list(config.get_other_option().values())))

if res.error_no == 0:
y = inp.task.flop / np.mean(res.costs)
else:
y = 0.0
return x, y
except Exception: # pylint: disable=broad-except
return None

def custom_callback(stopping_rounds, metric, fevals, evals=(), log_file=None,
maximize=False, verbose_eval=True):
Expand Down
4 changes: 3 additions & 1 deletion src/autotvm/touch_extractor.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,9 @@ void TouchExtractor::ExitItervar_() {
}
itervar_stack_.pop_back();

topdown_product_ /= itervar_map[var].length;
int64_t length = itervar_map[var].length;
if (length != 0)
topdown_product_ /= length;
int64_t bottomup_product = -1;
for (auto kv : itervar_map[var].touch_feature) {
bottomup_product = std::max(bottomup_product, kv.second.count * kv.second.reuse);
Expand Down