-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
git-svn-id: https://src.heuristiclab.com/svn/core/stable@17177 2abd9481-f8db-48e9-bd25-06bc13291c1b
- Loading branch information
Showing
8 changed files
with
431 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
158 changes: 158 additions & 0 deletions
158
...gleObjective/SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,158 @@ | ||
| #region License Information | ||
| /* HeuristicLab | ||
| * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL) | ||
| * | ||
| * This file is part of HeuristicLab. | ||
| * | ||
| * HeuristicLab is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * HeuristicLab is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| #endregion | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using HEAL.Attic; | ||
| using HeuristicLab.Common; | ||
| using HeuristicLab.Core; | ||
| using HeuristicLab.Data; | ||
| using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding; | ||
| using HeuristicLab.Parameters; | ||
| using HeuristicLab.PluginInfrastructure; | ||
|
|
||
| namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Classification { | ||
| [NonDiscoverableType] | ||
| [Item("Weighted Performance Measures Evaluator", "Calculates the quality of a symbolic classification solution based on three weighted measures(normalized mean squared error, false negative rate(1-sensitivity) and false positve rate(1-specificity)).")] | ||
| [StorableType("0772F316-5E12-4153-857E-8625069B4677")] | ||
| public class SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator : SymbolicClassificationSingleObjectiveEvaluator { | ||
| private const string NormalizedMeanSquaredErrorWeightingFactorParameterName = "NormalizedMeanSquaredErrorWeightingFactor"; | ||
| private const string FalseNegativeRateWeightingFactorParameterName = "FalseNegativeRateWeightingFactor"; | ||
| private const string FalsePositiveRateWeightingFactorParameterName = "FalsePositiveRateWeightingFactor"; | ||
| private const string ModelCreatorParameterName = "ModelCreator"; | ||
|
|
||
| public override bool Maximization { get { return false; } } | ||
|
|
||
| #region parameter properties | ||
| public IFixedValueParameter<DoubleValue> NormalizedMeanSquaredErrorWeightingFactorParameter { | ||
| get { return (IFixedValueParameter<DoubleValue>)Parameters[NormalizedMeanSquaredErrorWeightingFactorParameterName]; } | ||
| } | ||
| public IFixedValueParameter<DoubleValue> FalseNegativeRateWeightingFactorParameter { | ||
| get { return (IFixedValueParameter<DoubleValue>)Parameters[FalseNegativeRateWeightingFactorParameterName]; } | ||
| } | ||
| public IFixedValueParameter<DoubleValue> FalsePositiveRateWeightingFactorParameter { | ||
| get { return (IFixedValueParameter<DoubleValue>)Parameters[FalsePositiveRateWeightingFactorParameterName]; } | ||
| } | ||
| public IValueLookupParameter<ISymbolicClassificationModelCreator> ModelCreatorParameter { | ||
| get { return (IValueLookupParameter<ISymbolicClassificationModelCreator>)Parameters[ModelCreatorParameterName]; } | ||
| } | ||
| #endregion | ||
|
|
||
| public double NormalizedMeanSquaredErrorWeightingFactor { | ||
| get { return NormalizedMeanSquaredErrorWeightingFactorParameter.Value.Value; } | ||
| } | ||
| public double FalseNegativeRateWeightingFactor { | ||
| get { return FalseNegativeRateWeightingFactorParameter.Value.Value; } | ||
| } | ||
| public double FalsePositiveRateWeightingFactor { | ||
| get { return FalsePositiveRateWeightingFactorParameter.Value.Value; } | ||
| } | ||
|
|
||
| [StorableConstructor] | ||
| protected SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(StorableConstructorFlag _) : base(_) { } | ||
| protected SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator original, Cloner cloner) | ||
| : base(original, cloner) { | ||
| } | ||
| public override IDeepCloneable Clone(Cloner cloner) { | ||
| return new SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator(this, cloner); | ||
| } | ||
|
|
||
| public SymbolicClassificationSingleObjectiveWeightedPerformanceMeasuresEvaluator() | ||
| : base() { | ||
| Parameters.Add(new FixedValueParameter<DoubleValue>(NormalizedMeanSquaredErrorWeightingFactorParameterName, "The weighting factor of the normalized mean squared error.", new DoubleValue(1))); | ||
| Parameters.Add(new FixedValueParameter<DoubleValue>(FalseNegativeRateWeightingFactorParameterName, "The weighting factor of the false negative rate (1-sensitivity).", new DoubleValue(1))); | ||
| Parameters.Add(new FixedValueParameter<DoubleValue>(FalsePositiveRateWeightingFactorParameterName, "The weighting factor of the false positive rate (1-specificity).", new DoubleValue(1))); | ||
| Parameters.Add(new ValueLookupParameter<ISymbolicClassificationModelCreator>(ModelCreatorParameterName, "The model creator which is used during the evaluations.")); | ||
| } | ||
|
|
||
| public override IOperation InstrumentedApply() { | ||
| IEnumerable<int> rows = GenerateRowsToEvaluate(); | ||
| var tree = SymbolicExpressionTreeParameter.ActualValue; | ||
| var creator = ModelCreatorParameter.ActualValue; | ||
| var interpreter = SymbolicDataAnalysisTreeInterpreterParameter.ActualValue; | ||
| var estimationLimits = EstimationLimitsParameter.ActualValue; | ||
| var applyLinearScaling = ApplyLinearScalingParameter.ActualValue.Value; | ||
|
|
||
|
|
||
| double quality = Calculate(interpreter, tree, estimationLimits.Lower, estimationLimits.Upper, | ||
| ProblemDataParameter.ActualValue, rows, applyLinearScaling, creator, NormalizedMeanSquaredErrorWeightingFactor, FalseNegativeRateWeightingFactor, FalsePositiveRateWeightingFactor); | ||
| QualityParameter.ActualValue = new DoubleValue(quality); | ||
| return base.InstrumentedApply(); | ||
| } | ||
|
|
||
| public static double Calculate(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, ISymbolicExpressionTree tree, double lowerEstimationLimit, double upperEstimationLimit, IClassificationProblemData problemData, | ||
| IEnumerable<int> rows, bool applyLinearScaling, ISymbolicClassificationModelCreator modelCreator, double normalizedMeanSquaredErrorWeightingFactor, double falseNegativeRateWeightingFactor, double falsePositiveRateWeightingFactor) { | ||
| var estimatedValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, rows); | ||
| var targetClassValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows); | ||
| var boundedEstimatedValues = estimatedValues.LimitToRange(lowerEstimationLimit, upperEstimationLimit).ToArray(); | ||
| OnlineCalculatorError errorState; | ||
| double nmse; | ||
|
|
||
| //calculate performance measures | ||
| string positiveClassName = problemData.PositiveClass; | ||
| double[] classValues, thresholds; | ||
| IEnumerable<double> estimatedClassValues = null; | ||
| ISymbolicDiscriminantFunctionClassificationModel m; | ||
|
|
||
| var model = modelCreator.CreateSymbolicClassificationModel(problemData.TargetVariable, tree, interpreter, lowerEstimationLimit, upperEstimationLimit); | ||
| if ((m = model as ISymbolicDiscriminantFunctionClassificationModel) != null) { | ||
| m.ThresholdCalculator.Calculate(problemData, boundedEstimatedValues, targetClassValues, out classValues, out thresholds); | ||
| m.SetThresholdsAndClassValues(thresholds, classValues); | ||
| estimatedClassValues = m.GetEstimatedClassValues(boundedEstimatedValues); | ||
| } else { | ||
| model.RecalculateModelParameters(problemData, rows); | ||
| estimatedClassValues = model.GetEstimatedClassValues(problemData.Dataset, rows); | ||
| } | ||
|
|
||
| var performanceCalculator = new ClassificationPerformanceMeasuresCalculator(positiveClassName, problemData.GetClassValue(positiveClassName)); | ||
| performanceCalculator.Calculate(targetClassValues, estimatedClassValues); | ||
| if (performanceCalculator.ErrorState != OnlineCalculatorError.None) | ||
| return Double.NaN; | ||
| double falseNegativeRate = 1 - performanceCalculator.TruePositiveRate; | ||
| double falsePositiveRate = performanceCalculator.FalsePositiveRate; | ||
|
|
||
| if (applyLinearScaling) { | ||
| throw new NotSupportedException("The Weighted Performance Measures Evaluator does not suppport linear scaling!"); | ||
| } | ||
| nmse = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(targetClassValues, boundedEstimatedValues, out errorState); | ||
| if (errorState != OnlineCalculatorError.None) return Double.NaN; | ||
| return normalizedMeanSquaredErrorWeightingFactor * nmse + falseNegativeRateWeightingFactor * falseNegativeRate + falsePositiveRateWeightingFactor * falsePositiveRate; | ||
| } | ||
|
|
||
| public override double Evaluate(IExecutionContext context, ISymbolicExpressionTree tree, IClassificationProblemData problemData, IEnumerable<int> rows) { | ||
| SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = context; | ||
| EstimationLimitsParameter.ExecutionContext = context; | ||
| ApplyLinearScalingParameter.ExecutionContext = context; | ||
| ModelCreatorParameter.ExecutionContext = context; | ||
|
|
||
| double quality = Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper, | ||
| problemData, rows, ApplyLinearScalingParameter.ActualValue.Value, ModelCreatorParameter.ActualValue, NormalizedMeanSquaredErrorWeightingFactorParameter.Value.Value, FalseNegativeRateWeightingFactor, FalsePositiveRateWeightingFactor); | ||
|
|
||
| SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = null; | ||
| EstimationLimitsParameter.ExecutionContext = null; | ||
| ApplyLinearScalingParameter.ExecutionContext = null; | ||
| ModelCreatorParameter.ExecutionContext = null; | ||
|
|
||
| return quality; | ||
| } | ||
| } | ||
| } |
141 changes: 141 additions & 0 deletions
141
...ective/SymbolicClassificationSingleObjectiveWeightedResidualsMeanSquaredErrorEvaluator.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| #region License Information | ||
| /* HeuristicLab | ||
| * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL) | ||
| * | ||
| * This file is part of HeuristicLab. | ||
| * | ||
| * HeuristicLab is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * HeuristicLab is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| #endregion | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using HEAL.Attic; | ||
| using HeuristicLab.Common; | ||
| using HeuristicLab.Core; | ||
| using HeuristicLab.Data; | ||
| using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding; | ||
| using HeuristicLab.Parameters; | ||
| using HeuristicLab.PluginInfrastructure; | ||
|
|
||
| namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Classification { | ||
| [NonDiscoverableType] | ||
| [Item("Weighted Residuals Mean Squared Error Evaluator", @"A modified mean squared error evaluator that enables the possibility to weight residuals differently. | ||
| The first residual category belongs to estimated values which definitely belong to a specific class because the estimated value is located above the maximum or below the minimum of all the class values (DefiniteResidualsWeight). | ||
| The second residual category represents residuals which belong to the positive class whereby the estimated value is located between the positive and a negative class (PositiveClassResidualsWeight). | ||
| All other cases are represented by the third category (NegativeClassesResidualsWeight). | ||
| The weight gets multiplied to the squared error. Note that the Evaluator acts like a normal MSE-Evaluator if all the weights are set to 1.")] | ||
| [StorableType("A3193296-1A0F-46E2-8F43-22E2ED9CFFC5")] | ||
| public sealed class SymbolicClassificationSingleObjectiveWeightedResidualsMeanSquaredErrorEvaluator : SymbolicClassificationSingleObjectiveEvaluator { | ||
| private const string DefiniteResidualsWeightParameterName = "DefiniteResidualsWeight"; | ||
| private const string PositiveClassResidualsWeightParameterName = "PositiveClassResidualsWeight"; | ||
| private const string NegativeClassesResidualsWeightParameterName = "NegativeClassesResidualsWeight"; | ||
| [StorableConstructor] | ||
| private SymbolicClassificationSingleObjectiveWeightedResidualsMeanSquaredErrorEvaluator(StorableConstructorFlag _) : base(_) { } | ||
| private SymbolicClassificationSingleObjectiveWeightedResidualsMeanSquaredErrorEvaluator(SymbolicClassificationSingleObjectiveWeightedResidualsMeanSquaredErrorEvaluator original, Cloner cloner) | ||
| : base(original, cloner) { | ||
| } | ||
| public override IDeepCloneable Clone(Cloner cloner) { | ||
| return new SymbolicClassificationSingleObjectiveWeightedResidualsMeanSquaredErrorEvaluator(this, cloner); | ||
| } | ||
|
|
||
| public SymbolicClassificationSingleObjectiveWeightedResidualsMeanSquaredErrorEvaluator() | ||
| : base() { | ||
| Parameters.Add(new FixedValueParameter<DoubleValue>(DefiniteResidualsWeightParameterName, "Weight of residuals which definitely belong to a specific class because the estimated values is located above the maximum or below the minimum of all the class values.", new DoubleValue(1))); | ||
| Parameters.Add(new FixedValueParameter<DoubleValue>(PositiveClassResidualsWeightParameterName, "Weight of residuals which belong to the positive class whereby the estimated value is located between the positive and a negative class.", new DoubleValue(1))); | ||
| Parameters.Add(new FixedValueParameter<DoubleValue>(NegativeClassesResidualsWeightParameterName, "Weight of residuals which are not covered by the DefiniteResidualsWeight or the PositiveClassResidualsWeight.", new DoubleValue(1))); | ||
| } | ||
|
|
||
| #region parameter properties | ||
| public IFixedValueParameter<DoubleValue> DefiniteResidualsWeightParameter { | ||
| get { return (IFixedValueParameter<DoubleValue>)Parameters[DefiniteResidualsWeightParameterName]; } | ||
| } | ||
| public IFixedValueParameter<DoubleValue> PositiveClassResidualsWeightParameter { | ||
| get { return (IFixedValueParameter<DoubleValue>)Parameters[PositiveClassResidualsWeightParameterName]; } | ||
| } | ||
| public IFixedValueParameter<DoubleValue> NegativeClassesResidualsWeightParameter { | ||
| get { return (IFixedValueParameter<DoubleValue>)Parameters[NegativeClassesResidualsWeightParameterName]; } | ||
| } | ||
| #endregion | ||
|
|
||
| #region properties | ||
| public override bool Maximization { get { return false; } } | ||
|
|
||
| public double DefiniteResidualsWeight { | ||
| get { return DefiniteResidualsWeightParameter.Value.Value; } | ||
| } | ||
| public double PositiveClassResidualsWeight { | ||
| get { return PositiveClassResidualsWeightParameter.Value.Value; } | ||
| } | ||
| public double NegativeClassesResidualsWeight { | ||
| get { return NegativeClassesResidualsWeightParameter.Value.Value; } | ||
| } | ||
| #endregion | ||
|
|
||
| public override IOperation InstrumentedApply() { | ||
| IEnumerable<int> rows = GenerateRowsToEvaluate(); | ||
| var solution = SymbolicExpressionTreeParameter.ActualValue; | ||
| double quality = Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper, ProblemDataParameter.ActualValue, rows, ApplyLinearScalingParameter.ActualValue.Value, | ||
| DefiniteResidualsWeight, PositiveClassResidualsWeight, NegativeClassesResidualsWeight); | ||
| QualityParameter.ActualValue = new DoubleValue(quality); | ||
| return base.InstrumentedApply(); | ||
| } | ||
|
|
||
| public static double Calculate(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter, ISymbolicExpressionTree tree, double lowerEstimationLimit, double upperEstimationLimit, IClassificationProblemData problemData, IEnumerable<int> rows, bool applyLinearScaling, | ||
| double definiteResidualsWeight, double positiveClassResidualsWeight, double negativeClassesResidualsWeight) { | ||
| IEnumerable<double> estimatedValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, rows); | ||
| IEnumerable<double> targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows); | ||
| OnlineCalculatorError errorState; | ||
|
|
||
| double positiveClassValue = problemData.GetClassValue(problemData.PositiveClass); | ||
| //get class values min/max | ||
| double classValuesMin = problemData.ClassValues.ElementAtOrDefault(0); | ||
| double classValuesMax = classValuesMin; | ||
| foreach (double classValue in problemData.ClassValues) { | ||
| if (classValuesMin > classValue) classValuesMin = classValue; | ||
| if (classValuesMax < classValue) classValuesMax = classValue; | ||
| } | ||
|
|
||
| double quality; | ||
| if (applyLinearScaling) { | ||
| var calculator = new OnlineWeightedClassificationMeanSquaredErrorCalculator(positiveClassValue, classValuesMax, classValuesMin, | ||
| definiteResidualsWeight, positiveClassResidualsWeight, negativeClassesResidualsWeight); | ||
| CalculateWithScaling(targetValues, estimatedValues, lowerEstimationLimit, upperEstimationLimit, calculator, problemData.Dataset.Rows); | ||
| errorState = calculator.ErrorState; | ||
| quality = calculator.WeightedResidualsMeanSquaredError; | ||
| } else { | ||
| IEnumerable<double> boundedEstimatedValues = estimatedValues.LimitToRange(lowerEstimationLimit, upperEstimationLimit); | ||
| quality = OnlineWeightedClassificationMeanSquaredErrorCalculator.Calculate(targetValues, boundedEstimatedValues, positiveClassValue, classValuesMax, | ||
| classValuesMin, definiteResidualsWeight, positiveClassResidualsWeight, negativeClassesResidualsWeight, out errorState); | ||
| } | ||
| if (errorState != OnlineCalculatorError.None) return Double.NaN; | ||
| return quality; | ||
| } | ||
|
|
||
| public override double Evaluate(IExecutionContext context, ISymbolicExpressionTree tree, IClassificationProblemData problemData, IEnumerable<int> rows) { | ||
| SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = context; | ||
| EstimationLimitsParameter.ExecutionContext = context; | ||
| ApplyLinearScalingParameter.ExecutionContext = context; | ||
|
|
||
| double quality = Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper, problemData, rows, ApplyLinearScalingParameter.ActualValue.Value, DefiniteResidualsWeight, PositiveClassResidualsWeight, NegativeClassesResidualsWeight); | ||
|
|
||
| SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = null; | ||
| EstimationLimitsParameter.ExecutionContext = null; | ||
| ApplyLinearScalingParameter.ExecutionContext = null; | ||
|
|
||
| return quality; | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.