Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: use simpler converter for latin1 #1695

Merged
merged 1 commit into from
Dec 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dCommon/FdbToSqlite.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ std::string FdbToSqlite::Convert::ReadString(std::istream& cdClientBuffer) {
const auto readString = BinaryIO::ReadU8String(cdClientBuffer);

cdClientBuffer.seekg(prevPosition);
return GeneralUtils::Latin1ToWTF8(readString);
return GeneralUtils::Latin1ToUTF8(readString);
}

int32_t FdbToSqlite::Convert::SeekPointer(std::istream& cdClientBuffer) {
Expand Down
20 changes: 10 additions & 10 deletions dCommon/GeneralUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -167,11 +167,18 @@ std::u16string GeneralUtils::ASCIIToUTF16(const std::string_view string, const s
return ret;
}

std::string GeneralUtils::Latin1ToUTF8(const std::u8string_view string, const size_t size) {
std::string toReturn{};

//! Converts a (potentially-ill-formed) Latin1 string to UTF-8
for (const auto u : string) {
PushUTF8CodePoint(toReturn, u);
}
return toReturn;
}

//! Converts a (potentially-ill-formed) UTF-16 string to UTF-8
//! See: <http://simonsapin.github.io/wtf-8/#decoding-ill-formed-utf-16>
template<typename StringType>
std::string ToWTF8(const StringType string, const size_t size) {
std::string GeneralUtils::UTF16ToWTF8(const std::u16string_view string, const size_t size) {
const size_t newSize = MinSize(size, string);
std::string ret;
ret.reserve(newSize);
Expand All @@ -196,13 +203,6 @@ std::string ToWTF8(const StringType string, const size_t size) {

return ret;
}
std::string GeneralUtils::Latin1ToWTF8(const std::u8string_view string, const size_t size) {
return ToWTF8(string, size);
}

std::string GeneralUtils::UTF16ToWTF8(const std::u16string_view string, const size_t size) {
return ToWTF8(string, size);
}

bool GeneralUtils::CaseInsensitiveStringCompare(const std::string_view a, const std::string_view b) {
return std::equal(a.begin(), a.end(), b.begin(), b.end(), [](char a, char b) { return tolower(a) == tolower(b); });
Expand Down
2 changes: 1 addition & 1 deletion dCommon/GeneralUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ namespace GeneralUtils {
\param size A size to trim the string to. Default is SIZE_MAX (No trimming)
\return An UTF-8 representation of the string
*/
std::string Latin1ToWTF8(const std::u8string_view string, const size_t size = SIZE_MAX);
std::string Latin1ToUTF8(const std::u8string_view string, const size_t size = SIZE_MAX);

//! Converts a UTF-16 string to a UTF-8 string
/*!
Expand Down
Loading