Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feat: MetricEvent support UntypedMultiValues #1908

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+307 −17
Open

Feat: MetricEvent support UntypedMultiValues #1908

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
75ed7ff
add UntypedMultiValues
Takuka0311 Nov 22, 2024
6696eb1
polish
Takuka0311 Nov 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions core/models/MetricEvent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,50 @@ void MetricEvent::SetNameNoCopy(StringView name) {
mName = name;
}

double MetricEvent::GetMultiKeyValue(StringView key) {
if (UntypedMultiValues* multiValues = std::get_if<UntypedMultiValues>(&mValue)) {
if (multiValues->mValues.find(key) != multiValues->mValues.end()) {
return multiValues->mValues.at(key);
}
}
return 0;
}

bool MetricEvent::HasMultiKeyValue(StringView key) {
if (UntypedMultiValues* multiValues = std::get_if<UntypedMultiValues>(&mValue)) {
if (multiValues->mValues.find(key) != multiValues->mValues.end()) {
return true;
}
}
return false;
}

void MetricEvent::SetMultiKeyValue(const std::string& key, double val) {
SetMultiKeyValueNoCopy(GetSourceBuffer()->CopyString(key), val);
}

void MetricEvent::SetMultiKeyValue(StringView key, double val) {
SetMultiKeyValueNoCopy(GetSourceBuffer()->CopyString(key), val);
}

void MetricEvent::SetMultiKeyValueNoCopy(const StringBuffer& key, double val) {
SetMultiKeyValueNoCopy(StringView(key.data, key.size), val);
}

void MetricEvent::SetMultiKeyValueNoCopy(StringView key, double val) {
if (UntypedMultiValues* multiValues = std::get_if<UntypedMultiValues>(&mValue)) {
multiValues->mValues[key] = val;
} else if (Is<monostate>()) {
mValue = UntypedMultiValues{{{key, val}}};
}
}

void MetricEvent::DelMultiKeyValue(StringView key) {
if (UntypedMultiValues* multiValues = std::get_if<UntypedMultiValues>(&mValue)) {
multiValues->mValues.erase(key);
}
}

StringView MetricEvent::GetTag(StringView key) const {
auto it = mTags.mInner.find(key);
if (it != mTags.mInner.end()) {
Expand Down
8 changes: 8 additions & 0 deletions core/models/MetricEvent.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,14 @@ class MetricEvent : public PipelineEvent {
mValue = T{std::forward<Args>(args)...};
}

double GetMultiKeyValue(StringView key);
bool HasMultiKeyValue(StringView key);
void SetMultiKeyValue(const std::string& key, double val);
void SetMultiKeyValue(StringView key, double val);
void SetMultiKeyValueNoCopy(const StringBuffer& key, double val);
void SetMultiKeyValueNoCopy(StringView key, double val);
void DelMultiKeyValue(StringView key);
Takuka0311 marked this conversation as resolved.
Show resolved Hide resolved

StringView GetTag(StringView key) const;
bool HasTag(StringView key) const;
void SetTag(StringView key, StringView val);
Expand Down
34 changes: 34 additions & 0 deletions core/models/MetricValue.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@ using namespace std;

namespace logtail {

size_t UntypedMultiValues::DataSize() const {
size_t totalSize = sizeof(UntypedMultiValues);
for (const auto& pair : mValues) {
totalSize += pair.first.size() + sizeof(pair.second);
}
return totalSize;
}

size_t DataSize(const MetricValue& value) {
return visit(
[](auto&& arg) {
Expand All @@ -42,6 +50,24 @@ void UntypedSingleValue::FromJson(const Json::Value& value) {
mValue = value.asFloat();
}

Json::Value UntypedMultiValues::ToJson() const {
Json::Value res;
for (auto metric : mValues) {
res[metric.first.to_string()] = metric.second;
}
return res;
}

void UntypedMultiValues::FromJson(const Json::Value& value) {
mValues.clear();
for (Json::Value::const_iterator itr = value.begin(); itr != value.end(); ++itr) {
if (itr->asDouble()) {
StringBuffer s = mSourceBuffer->CopyString(itr.key().asString());
mValues[StringView(s.data, s.size)] = itr->asDouble();
}
}
}

Json::Value MetricValueToJson(const MetricValue& value) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MetricValueToJson、JsonToMetricValue都需要ut覆盖

Json::Value res;
visit(
Expand All @@ -50,6 +76,9 @@ Json::Value MetricValueToJson(const MetricValue& value) {
if constexpr (is_same_v<T, UntypedSingleValue>) {
res["type"] = "untyped_single_value";
res["detail"] = get<UntypedSingleValue>(value).ToJson();
} else if constexpr (is_same_v<T, UntypedMultiValues>) {
res["type"] = "untyped_multi_values";
res["detail"] = get<UntypedMultiValues>(value).ToJson();
} else if constexpr (is_same_v<T, monostate>) {
res["type"] = "unknown";
}
Expand All @@ -63,6 +92,11 @@ MetricValue JsonToMetricValue(const string& type, const Json::Value& detail) {
UntypedSingleValue v;
v.FromJson(detail);
return v;
} else if (type == "untyped_multi_values") {
UntypedMultiValues v;
v.mSourceBuffer = std::make_shared<SourceBuffer>();
v.FromJson(detail);
return v;
} else {
return MetricValue();
}
Expand Down
19 changes: 18 additions & 1 deletion core/models/MetricValue.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,18 @@
#pragma once

#include <variant>
#include <map>

#ifdef APSARA_UNIT_TEST_MAIN
#include <json/json.h>

#include <string>

#include "common/memory/SourceBuffer.h"
#endif

#include "models/StringView.h"

namespace logtail {

struct UntypedSingleValue {
Expand All @@ -37,7 +42,19 @@ struct UntypedSingleValue {
#endif
};

using MetricValue = std::variant<std::monostate, UntypedSingleValue>;
struct UntypedMultiValues {
Takuka0311 marked this conversation as resolved.
Show resolved Hide resolved
std::map<StringView, double> mValues;

size_t DataSize() const;

#ifdef APSARA_UNIT_TEST_MAIN
std::shared_ptr<SourceBuffer> mSourceBuffer;
Takuka0311 marked this conversation as resolved.
Show resolved Hide resolved
Json::Value ToJson() const;
void FromJson(const Json::Value& value);
#endif
};

using MetricValue = std::variant<std::monostate, UntypedSingleValue, UntypedMultiValues>;

size_t DataSize(const MetricValue& value);

Expand Down
162 changes: 150 additions & 12 deletions core/unittest/models/MetricEventUnittest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,17 @@ namespace logtail {
class MetricEventUnittest : public ::testing::Test {
public:
void TestName();
void TestValue();
void TestUntypedSingleValue();
void TestUntypedMultiValues();
void TestTag();
void TestSize();
void TestUntypedSingleValueSize();
void TestUntypedMultiValuesSize();
void TestReset();
void TestToJson();
void TestFromJson();
void TestTypeConflict();
void TestUntypedSingleValueToJson();
void TestUntypedMultiValuesToJson();
void TestUntypedSingleValueFromJson();
void TestUntypedMultiValuesFromJson();
void TestTagsIterator();

protected:
Expand All @@ -51,7 +56,7 @@ void MetricEventUnittest::TestName() {
APSARA_TEST_EQUAL("test", mMetricEvent->GetName().to_string());
}

void MetricEventUnittest::TestValue() {
void MetricEventUnittest::TestUntypedSingleValue() {
mMetricEvent->SetValue(UntypedSingleValue{10.0});
APSARA_TEST_TRUE(mMetricEvent->Is<UntypedSingleValue>());
APSARA_TEST_EQUAL(10.0, mMetricEvent->GetValue<UntypedSingleValue>()->mValue);
Expand All @@ -61,6 +66,25 @@ void MetricEventUnittest::TestValue() {
APSARA_TEST_EQUAL(100.0, mMetricEvent->GetValue<UntypedSingleValue>()->mValue);
}

void MetricEventUnittest::TestUntypedMultiValues() {
mMetricEvent->SetValue(UntypedMultiValues{{{"test-1", 10.0}, {"test-2", 2.0}}});
APSARA_TEST_TRUE(mMetricEvent->Is<UntypedMultiValues>());
APSARA_TEST_EQUAL(10.0, mMetricEvent->GetValue<UntypedMultiValues>()->mValues.at("test-1"));
APSARA_TEST_EQUAL(2.0, mMetricEvent->GetValue<UntypedMultiValues>()->mValues.at("test-2"));

map<StringView, double> metrics({{"test-3", 15.0}, {"test-4", 24.0}});
mMetricEvent->SetValue<UntypedMultiValues>(metrics);
APSARA_TEST_TRUE(mMetricEvent->Is<UntypedMultiValues>());
APSARA_TEST_EQUAL(15.0, mMetricEvent->GetValue<UntypedMultiValues>()->mValues.at("test-3"));
APSARA_TEST_EQUAL(24.0, mMetricEvent->GetValue<UntypedMultiValues>()->mValues.at("test-4"));

mMetricEvent->SetMultiKeyValue(string("test-1"), 6.0);
APSARA_TEST_EQUAL(6.0, mMetricEvent->GetMultiKeyValue("test-1"));

mMetricEvent->DelMultiKeyValue("test-4");
APSARA_TEST_EQUAL(0, mMetricEvent->GetMultiKeyValue("test-4"));
}

void MetricEventUnittest::TestTag() {
{
string key = "key1";
Expand Down Expand Up @@ -106,7 +130,7 @@ void MetricEventUnittest::TestTag() {
}
}

void MetricEventUnittest::TestSize() {
void MetricEventUnittest::TestUntypedSingleValueSize() {
size_t basicSize = sizeof(time_t) + sizeof(long) + sizeof(UntypedSingleValue) + sizeof(map<StringView, StringView>);
mMetricEvent->SetName("test");
basicSize += 4;
Expand All @@ -126,6 +150,39 @@ void MetricEventUnittest::TestSize() {
APSARA_TEST_EQUAL(basicSize, mMetricEvent->DataSize());
}

void MetricEventUnittest::TestUntypedMultiValuesSize() {
mMetricEvent->SetName("test");
mMetricEvent->SetValue(UntypedMultiValues{});
size_t basicSize = sizeof(time_t) + sizeof(long) + sizeof(UntypedMultiValues) + sizeof(map<StringView, StringView>);
basicSize += 4;

// add tag, and key not existed
mMetricEvent->SetTag(string("key1"), string("a"));
APSARA_TEST_EQUAL(basicSize + 5U, mMetricEvent->DataSize());

// add tag, and key existed
mMetricEvent->SetTag(string("key1"), string("bb"));
APSARA_TEST_EQUAL(basicSize + 6U, mMetricEvent->DataSize());

// delete tag
mMetricEvent->DelTag(string("key1"));
APSARA_TEST_EQUAL(basicSize, mMetricEvent->DataSize());

// add multi values, and key not existed
mMetricEvent->SetMultiKeyValue(string("test-1"), 5.0);
basicSize += 14;
APSARA_TEST_EQUAL(basicSize, mMetricEvent->DataSize());

// add multi values, and key existed
mMetricEvent->SetMultiKeyValue(string("test-1"), 99.0);
APSARA_TEST_EQUAL(basicSize, mMetricEvent->DataSize());

// delete multi values
mMetricEvent->DelMultiKeyValue("test-1");
basicSize -= 14;
APSARA_TEST_EQUAL(basicSize, mMetricEvent->DataSize());
}

void MetricEventUnittest::TestReset() {
mMetricEvent->SetTimestamp(12345678901);
mMetricEvent->SetName("test");
Expand All @@ -139,7 +196,21 @@ void MetricEventUnittest::TestReset() {
APSARA_TEST_EQUAL(mMetricEvent->TagsEnd(), mMetricEvent->TagsBegin());
}

void MetricEventUnittest::TestToJson() {
void MetricEventUnittest::TestTypeConflict() {
mMetricEvent->SetTimestamp(12345678901);
mMetricEvent->SetName("test");
mMetricEvent->SetTag(string("key1"), string("value1"));

mMetricEvent->SetMultiKeyValue(string("test-1"), 6.0);
APSARA_TEST_EQUAL(6.0, mMetricEvent->GetMultiKeyValue("test-1"));

mMetricEvent->SetValue(UntypedSingleValue{10.0});

mMetricEvent->SetMultiKeyValue(string("test-1"), 6.0);
APSARA_TEST_EQUAL(0, mMetricEvent->GetMultiKeyValue("test-1"));
}

void MetricEventUnittest::TestUntypedSingleValueToJson() {
mMetricEvent->SetTimestamp(12345678901, 0);
mMetricEvent->SetName("test");
mMetricEvent->SetValue(UntypedSingleValue{10.0});
Expand All @@ -166,7 +237,37 @@ void MetricEventUnittest::TestToJson() {
APSARA_TEST_TRUE(eventJson == res);
}

void MetricEventUnittest::TestFromJson() {
void MetricEventUnittest::TestUntypedMultiValuesToJson() {
mMetricEvent->SetTimestamp(12345678901, 0);
mMetricEvent->SetName("test");
mMetricEvent->SetValue(UntypedMultiValues{{{"test-1", 10.0}, {"test-2", 2.0}}});
mMetricEvent->SetTag(string("key1"), string("value1"));
Json::Value res = mMetricEvent->ToJson();

Json::Value eventJson;
string eventStr = R"({
"name": "test",
"tags": {
"key1": "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"type" : 2,
"value": {
"type": "untyped_multi_values",
"detail": {
"test-1": 10.0,
"test-2": 2.0
}
}
})";
string errorMsg;
ParseJsonTable(eventStr, eventJson, errorMsg);

APSARA_TEST_TRUE(eventJson == res);
}

void MetricEventUnittest::TestUntypedSingleValueFromJson() {
Json::Value eventJson;
string eventStr = R"({
"name": "test",
Expand All @@ -192,6 +293,38 @@ void MetricEventUnittest::TestFromJson() {
APSARA_TEST_EQUAL("value1", mMetricEvent->GetTag("key1").to_string());
}

void MetricEventUnittest::TestUntypedMultiValuesFromJson() {
Json::Value eventJson;
string eventStr = R"({
"name": "test",
"tags": {
"key1": "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
"value": {
"type": "untyped_multi_values",
"detail": {
"test-1": 10.0,
"test-2": 2.0
}
}
})";
string errorMsg;
ParseJsonTable(eventStr, eventJson, errorMsg);
mMetricEvent->FromJson(eventJson);

APSARA_TEST_EQUAL(12345678901, mMetricEvent->GetTimestamp());
APSARA_TEST_EQUAL(0L, mMetricEvent->GetTimestampNanosecond().value());
APSARA_TEST_EQUAL("test", mMetricEvent->GetName());
APSARA_TEST_TRUE(mMetricEvent->Is<UntypedMultiValues>());
APSARA_TEST_EQUAL(10.0, mMetricEvent->GetValue<UntypedMultiValues>()->mValues.at("test-1"));
APSARA_TEST_EQUAL(2.0, mMetricEvent->GetValue<UntypedMultiValues>()->mValues.at("test-2"));
APSARA_TEST_EQUAL(10.0, mMetricEvent->GetMultiKeyValue("test-1"));
APSARA_TEST_EQUAL(2.0, mMetricEvent->GetMultiKeyValue("test-2"));
APSARA_TEST_EQUAL("value1", mMetricEvent->GetTag("key1").to_string());
}

void MetricEventUnittest::TestTagsIterator() {
string key1 = "key1";
string value1 = "value1";
Expand All @@ -216,12 +349,17 @@ void MetricEventUnittest::TestTagsIterator() {
}

UNIT_TEST_CASE(MetricEventUnittest, TestName)
UNIT_TEST_CASE(MetricEventUnittest, TestValue)
UNIT_TEST_CASE(MetricEventUnittest, TestUntypedSingleValue)
UNIT_TEST_CASE(MetricEventUnittest, TestUntypedMultiValues)
UNIT_TEST_CASE(MetricEventUnittest, TestTag)
UNIT_TEST_CASE(MetricEventUnittest, TestSize)
UNIT_TEST_CASE(MetricEventUnittest, TestUntypedSingleValueSize)
UNIT_TEST_CASE(MetricEventUnittest, TestUntypedMultiValuesSize)
UNIT_TEST_CASE(MetricEventUnittest, TestReset)
UNIT_TEST_CASE(MetricEventUnittest, TestToJson)
UNIT_TEST_CASE(MetricEventUnittest, TestFromJson)
UNIT_TEST_CASE(MetricEventUnittest, TestTypeConflict)
UNIT_TEST_CASE(MetricEventUnittest, TestUntypedSingleValueToJson)
UNIT_TEST_CASE(MetricEventUnittest, TestUntypedMultiValuesToJson)
UNIT_TEST_CASE(MetricEventUnittest, TestUntypedSingleValueFromJson)
UNIT_TEST_CASE(MetricEventUnittest, TestUntypedMultiValuesFromJson)
UNIT_TEST_CASE(MetricEventUnittest, TestTagsIterator)

} // namespace logtail
Expand Down
Loading
Loading