Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] Support for multi-parameter loss functions in derivative aggregation and finding best splits #993

Merged
merged 14 commits into from
Feb 17, 2020
Merged

[ML] Support for multi-parameter loss functions in derivative aggregation and finding best splits #993

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
c6df959
Extend derivatives aggregation and solving for best split to multi-pa…
tveasey Feb 10, 2020
570e0f3
Rework derivatives storage to cut down performance impact of heap all…
tveasey Feb 11, 2020
8240a0a
Optimize computing derivatives difference
tveasey Feb 11, 2020
2db432b
Couple more performance tweaks
tveasey Feb 11, 2020
da33060
Further tweak
tveasey Feb 11, 2020
deea270
Improve comment
tveasey Feb 11, 2020
8e928e1
Test fall out and some memory estimation and usage improvements
tveasey Feb 12, 2020
f19e74c
Increase memory limit for windows
tveasey Feb 12, 2020
ac46718
Relax threshold for Windows
tveasey Feb 13, 2020
e1556ce
Explain the map function
tveasey Feb 14, 2020
3009f42
Improve explanations in computeBestSplitStatistics
tveasey Feb 14, 2020
e4cb871
Comment tweaks
tveasey Feb 14, 2020
4d9b0ac
Review comment
tveasey Feb 14, 2020
accd1a8
Merge branch 'master' into multiclass-4
tveasey Feb 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 57 additions & 29 deletions include/maths/CBoostedTree.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,20 @@ class CEncodedDataFrameRowRef;
namespace boosted_tree_detail {
class MATHS_EXPORT CArgMinLossImpl {
public:
using TDouble1Vec = core::CSmallVector<double, 1>;
using TDoubleVector = CDenseVector<double>;
using TMemoryMappedFloatVector = CMemoryMappedDenseVector<CFloatStorage>;

public:
CArgMinLossImpl(double lambda);
virtual ~CArgMinLossImpl() = default;

virtual std::unique_ptr<CArgMinLossImpl> clone() const = 0;
virtual bool nextPass() = 0;
virtual void add(TDouble1Vec prediction, double actual, double weight = 1.0) = 0;
virtual void add(const TMemoryMappedFloatVector& prediction,
double actual,
double weight = 1.0) = 0;
virtual void merge(const CArgMinLossImpl& other) = 0;
virtual TDouble1Vec value() const = 0;
virtual TDoubleVector value() const = 0;

protected:
double lambda() const;
Expand All @@ -61,9 +64,9 @@ class MATHS_EXPORT CArgMinMseImpl final : public CArgMinLossImpl {
CArgMinMseImpl(double lambda);
std::unique_ptr<CArgMinLossImpl> clone() const override;
bool nextPass() override;
void add(TDouble1Vec prediction, double actual, double weight = 1.0) override;
void add(const TMemoryMappedFloatVector& prediction, double actual, double weight = 1.0) override;
void merge(const CArgMinLossImpl& other) override;
TDouble1Vec value() const override;
TDoubleVector value() const override;

private:
using TMeanAccumulator = CBasicStatistics::SSampleMean<double>::TAccumulator;
Expand All @@ -79,14 +82,14 @@ class MATHS_EXPORT CArgMinLogisticImpl final : public CArgMinLossImpl {
CArgMinLogisticImpl(double lambda);
std::unique_ptr<CArgMinLossImpl> clone() const override;
bool nextPass() override;
void add(TDouble1Vec prediction, double actual, double weight = 1.0) override;
void add(const TMemoryMappedFloatVector& prediction, double actual, double weight = 1.0) override;
void merge(const CArgMinLossImpl& other) override;
TDouble1Vec value() const override;
TDoubleVector value() const override;

private:
using TMinMaxAccumulator = CBasicStatistics::CMinMax<double>;
using TVector = CVectorNx1<double, 2>;
using TVectorVec = std::vector<TVector>;
using TDoubleVector2x1 = CVectorNx1<double, 2>;
using TDoubleVector2x1Vec = std::vector<TDoubleVector2x1>;

private:
std::size_t bucket(double prediction) const {
Expand All @@ -108,8 +111,8 @@ class MATHS_EXPORT CArgMinLogisticImpl final : public CArgMinLossImpl {
private:
std::size_t m_CurrentPass = 0;
TMinMaxAccumulator m_PredictionMinMax;
TVector m_CategoryCounts;
TVectorVec m_BucketCategoryCounts;
TDoubleVector2x1 m_CategoryCounts;
TDoubleVector2x1Vec m_BucketCategoryCounts;
};
}

Expand All @@ -118,7 +121,8 @@ namespace boosted_tree {
//! \brief Computes the leaf value which minimizes the loss function.
class MATHS_EXPORT CArgMinLoss {
public:
using TDouble1Vec = core::CSmallVector<double, 1>;
using TDoubleVector = CDenseVector<double>;
using TMemoryMappedFloatVector = CMemoryMappedDenseVector<CFloatStorage>;

public:
CArgMinLoss(const CArgMinLoss& other);
Expand All @@ -133,7 +137,7 @@ class MATHS_EXPORT CArgMinLoss {
bool nextPass() const;

//! Update with a point prediction and actual value.
void add(TDouble1Vec prediction, double actual, double weight = 1.0);
void add(const TMemoryMappedFloatVector& prediction, double actual, double weight = 1.0);

//! Get the minimiser over the predictions and actual values added to both
//! this and \p other.
Expand All @@ -144,7 +148,7 @@ class MATHS_EXPORT CArgMinLoss {
//!
//! Formally, returns \f$x^* = arg\min_x\{\sum_i{L(p_i + x, a_i)}\}\f$
//! for predictions and actuals \f$p_i\f$ and \f$a_i\f$, respectively.
TDouble1Vec value() const;
TDoubleVector value() const;

private:
using TArgMinLossImplUPtr = std::unique_ptr<boosted_tree_detail::CArgMinLossImpl>;
Expand All @@ -161,7 +165,9 @@ class MATHS_EXPORT CArgMinLoss {
//! \brief Defines the loss function for the regression problem.
class MATHS_EXPORT CLoss {
public:
using TDouble1Vec = core::CSmallVector<double, 1>;
using TDoubleVector = CDenseVector<double>;
using TMemoryMappedFloatVector = CMemoryMappedDenseVector<CFloatStorage>;
using TWriter = std::function<void(std::size_t, double)>;

public:
virtual ~CLoss() = default;
Expand All @@ -170,17 +176,23 @@ class MATHS_EXPORT CLoss {
//! The number of parameters to the loss function.
virtual std::size_t numberParameters() const = 0;
//! The value of the loss function.
virtual double value(TDouble1Vec prediction, double actual, double weight = 1.0) const = 0;
virtual double value(const TMemoryMappedFloatVector& prediction,
double actual,
double weight = 1.0) const = 0;
//! The gradient of the loss function.
virtual TDouble1Vec
gradient(TDouble1Vec prediction, double actual, double weight = 1.0) const = 0;
virtual void gradient(const TMemoryMappedFloatVector& prediction,
double actual,
TWriter writer,
double weight = 1.0) const = 0;
//! The Hessian of the loss function (flattened).
virtual TDouble1Vec
curvature(TDouble1Vec prediction, double actual, double weight = 1.0) const = 0;
virtual void curvature(const TMemoryMappedFloatVector& prediction,
double actual,
TWriter writer,
double weight = 1.0) const = 0;
//! Returns true if the loss curvature is constant.
virtual bool isCurvatureConstant() const = 0;
//! Transforms a prediction from the forest to the target space.
virtual TDouble1Vec transform(TDouble1Vec prediction) const = 0;
virtual TDoubleVector transform(const TMemoryMappedFloatVector& prediction) const = 0;
//! Get an object which computes the leaf value that minimises loss.
virtual CArgMinLoss minimizer(double lambda) const = 0;
//! Get the name of the loss function
Expand All @@ -198,11 +210,19 @@ class MATHS_EXPORT CMse final : public CLoss {
public:
std::unique_ptr<CLoss> clone() const override;
std::size_t numberParameters() const override;
double value(TDouble1Vec prediction, double actual, double weight = 1.0) const override;
TDouble1Vec gradient(TDouble1Vec prediction, double actual, double weight = 1.0) const override;
TDouble1Vec curvature(TDouble1Vec prediction, double actual, double weight = 1.0) const override;
double value(const TMemoryMappedFloatVector& prediction,
double actual,
double weight = 1.0) const override;
void gradient(const TMemoryMappedFloatVector& prediction,
double actual,
TWriter writer,
double weight = 1.0) const override;
void curvature(const TMemoryMappedFloatVector& prediction,
double actual,
TWriter writer,
double weight = 1.0) const override;
bool isCurvatureConstant() const override;
TDouble1Vec transform(TDouble1Vec prediction) const override;
TDoubleVector transform(const TMemoryMappedFloatVector& prediction) const override;
CArgMinLoss minimizer(double lambda) const override;
const std::string& name() const override;
};
Expand All @@ -223,11 +243,19 @@ class MATHS_EXPORT CBinomialLogistic final : public CLoss {
public:
std::unique_ptr<CLoss> clone() const override;
std::size_t numberParameters() const override;
double value(TDouble1Vec prediction, double actual, double weight = 1.0) const override;
TDouble1Vec gradient(TDouble1Vec prediction, double actual, double weight = 1.0) const override;
TDouble1Vec curvature(TDouble1Vec prediction, double actual, double weight = 1.0) const override;
double value(const TMemoryMappedFloatVector& prediction,
double actual,
double weight = 1.0) const override;
void gradient(const TMemoryMappedFloatVector& prediction,
double actual,
TWriter writer,
double weight = 1.0) const override;
void curvature(const TMemoryMappedFloatVector& prediction,
double actual,
TWriter writer,
double weight = 1.0) const override;
bool isCurvatureConstant() const override;
TDouble1Vec transform(TDouble1Vec prediction) const override;
TDoubleVector transform(const TMemoryMappedFloatVector& prediction) const override;
CArgMinLoss minimizer(double lambda) const override;
const std::string& name() const override;
};
Expand Down
Loading