Skip to content

Commit

Permalink
Allow TiFlash server run without a config file (#7251)
Browse files Browse the repository at this point in the history 
ref #4809
  • Loading branch information
@breezewish
breezewish authored Apr 10, 2023
1 parent 1237e95 commit e0c82c0
Show file tree
Hide file tree
Showing 25 changed files with 341 additions and 177 deletions.
2 changes: 1 addition & 1 deletion contrib/tiflash-proxy
Submodule tiflash-proxy updated 5 files
+5 −1 proxy_components/proxy_ffi/src/basic_ffi_impls.rs
+780 −0 proxy_components/proxy_server/src/common.rs
+236 −0 proxy_components/proxy_server/src/common_override.rs
+4 −0 proxy_components/proxy_server/src/lib.rs
+168 −671 proxy_components/proxy_server/src/run.rs
2 changes: 1 addition & 1 deletion dbms/src/Common/ComputeLabelHolder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ void ComputeLabelHolder::init(const Poco::Util::LayeredConfiguration & conf)
std::lock_guard lock(mu);
RUNTIME_ASSERT(!label_got, log, "Can't init after label got");
cluster_id = conf.getString(cluster_id_key, "unknown");
auto service_addr = conf.getString("flash.service_addr", "unknown");
auto service_addr = conf.getString("flash.service_addr", "0.0.0.0:3930");
std::replace(service_addr.begin(), service_addr.end(), ':', '_');
std::replace(service_addr.begin(), service_addr.end(), '.', '_');
process_id = fmt::format("compute_{}_{}", service_addr, microsecondsUTC());
Expand Down
8 changes: 0 additions & 8 deletions dbms/src/Common/Config/ConfigProcessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,3 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig()

return LoadedConfig{configuration, false, config_doc};
}

void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
{
std::ofstream out(preprocessed_path);
cpptoml::toml_writer writer(out);
loaded_config.preprocessed_conf->accept(std::move(writer));
out.close();
}
2 changes: 0 additions & 2 deletions dbms/src/Common/Config/ConfigProcessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ class ConfigProcessor

LoadedConfig loadConfig();

void savePreprocessedConfig(const LoadedConfig & loaded_config);

public:
/// Is the file named as result of config preprocessing, not as original files.
static bool isPreprocessedFile(const std::string & config_path);
Expand Down
6 changes: 3 additions & 3 deletions dbms/src/Common/Config/ConfigReloader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error)
}
}

if (force || new_files.isDifferOrNewerThan(files) || config_object_updated)
if (force
|| (new_files.valid() && new_files.isDifferOrNewerThan(files))
|| config_object_updated)
{
ConfigProcessor config_processor(path);
ConfigProcessor::LoadedConfig loaded_config;
Expand All @@ -103,8 +105,6 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error)
return;
}

config_processor.savePreprocessedConfig(loaded_config);

/** We should remember last modification time if and only if config was sucessfully loaded
* Otherwise a race condition could occur during config files update:
* File is contain raw (and non-valid) data, therefore config is not applied.
Expand Down
5 changes: 5 additions & 0 deletions dbms/src/Common/FileChangesTracker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ struct FilesChangesTracker
{
std::set<FileWithTimestamp> files;

bool valid() const
{
return !files.empty();
}

void addIfExists(const std::string & path)
{
if (!path.empty() && Poco::File(path).exists())
Expand Down
254 changes: 193 additions & 61 deletions dbms/src/Common/UnifiedLogFormatter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,55 +23,100 @@
#include <chrono>
#include <vector>

#define RESET_FONT "\033[0m"
#define BOLD_FONT "\033[1m"
#define BLACK "\033[30m"
#define RED "\033[31m"
#define GREEN "\033[32m"
#define YELLOW "\033[33m"
#define BLUE "\033[34m"
#define MAGENTA "\033[35m"
#define CYAN "\033[36m"
#define WHITE "\033[37m"
#define BRIGHT_BLACK "\033[90m"
#define BRIGHT_RED "\033[91m"
#define BRIGHT_GREEN "\033[92m"
#define BRIGHT_YELLOW "\033[93m"
#define BRIGHT_BLUE "\033[94m"
#define BRIGHT_MAGENTA "\033[95m"
#define BRIGHT_CYAN "\033[96m"
#define BRIGHT_WHITE "\033[97m"

namespace DB
{
void UnifiedLogFormatter::format(const Poco::Message & msg, std::string & text)

enum class JsonEncodeKind
{
FmtBuffer buf;
/**
* No need to encode, just copy the text
*/
DirectCopy,

// Timestamp
{
buf.append('[');
writeTimestamp(buf);
buf.append("] ");
}
// Priority
{
buf.append('[');
writePriority(buf, msg.getPriority());
buf.append("] ");
}
// Source File
{
if (unlikely(!msg.getSourceFile()))
buf.append("[<unknown>] ");
else
buf.fmtAppend(FMT_COMPILE("[{}:{}] "), msg.getSourceFile(), msg.getSourceLine());
}
// Message
{
buf.append('[');
writeEscapedString(buf, msg.getText());
buf.append("] ");
}
// Source and Identifiers
/**
* Add double quotes around the text is sufficient
*/
AddQuoteAndCopy,

/**
* Need full JSON string encode
*/
Encode,
};

void writePriorityColor(FmtBuffer & buf, const Poco::Message::Priority & priority)
{
switch (priority)
{
const std::string & source = msg.getSource();
if (!source.empty())
{
buf.append("[source=");
writeEscapedString(buf, source);
buf.append("] ");
}
case Poco::Message::Priority::PRIO_TRACE:
buf.append(BRIGHT_BLACK);
break;
case Poco::Message::Priority::PRIO_DEBUG:
buf.append(BRIGHT_BLACK);
break;
case Poco::Message::Priority::PRIO_INFORMATION:
buf.append(BRIGHT_BLUE);
break;
case Poco::Message::Priority::PRIO_WARNING:
buf.append(YELLOW);
break;
case Poco::Message::Priority::PRIO_ERROR:
buf.append(RED);
break;
case Poco::Message::Priority::PRIO_FATAL:
buf.append(RED);
break;
case Poco::Message::Priority::PRIO_CRITICAL:
buf.append(RED);
break;
case Poco::Message::Priority::PRIO_NOTICE:
break;
default:
break;
}
// Thread ID
}

void writeMessageColor(FmtBuffer & buf, const Poco::Message::Priority & priority)
{
switch (priority)
{
buf.fmtAppend(FMT_COMPILE("[thread_id={}]"), Poco::ThreadNumber::get());
case Poco::Message::Priority::PRIO_WARNING:
buf.append(YELLOW);
break;
case Poco::Message::Priority::PRIO_ERROR:
buf.append(RED);
break;
case Poco::Message::Priority::PRIO_FATAL:
buf.append(RED);
break;
case Poco::Message::Priority::PRIO_CRITICAL:
buf.append(RED);
break;
default:
break;
}
text = buf.toString();
}

void UnifiedLogFormatter::writePriority(FmtBuffer & buf, const Poco::Message::Priority & priority)
void writePriority(FmtBuffer & buf, const Poco::Message::Priority & priority)
{
switch (priority)
{
Expand Down Expand Up @@ -105,7 +150,7 @@ void UnifiedLogFormatter::writePriority(FmtBuffer & buf, const Poco::Message::Pr
}
}

void UnifiedLogFormatter::writeTimestamp(FmtBuffer & buf)
void writeTimestamp(FmtBuffer & buf)
{
// The format is "yyyy/MM/dd HH:mm:ss.SSS ZZZZZ"
auto time_point = std::chrono::system_clock::now();
Expand Down Expand Up @@ -151,26 +196,7 @@ void UnifiedLogFormatter::writeTimestamp(FmtBuffer & buf)
buf.fmtAppend(FMT_COMPILE("{0:02d}:{1:02d}"), offset_tm->tm_hour, offset_tm->tm_min);
}

void UnifiedLogFormatter::writeEscapedString(FmtBuffer & buf, const std::string & str)
{
auto encode_kind = needJsonEncode(str);
switch (encode_kind)
{
case JsonEncodeKind::DirectCopy:
buf.append(str);
break;
case JsonEncodeKind::AddQuoteAndCopy:
buf.append('"');
buf.append(str);
buf.append('"');
break;
case JsonEncodeKind::Encode:
writeJSONString(buf, str);
break;
}
}

UnifiedLogFormatter::JsonEncodeKind UnifiedLogFormatter::needJsonEncode(const std::string & src)
JsonEncodeKind needJsonEncode(const std::string & src)
{
bool needs_quote = false;
bool json_encode_cannot_copy = false;
Expand All @@ -192,7 +218,7 @@ UnifiedLogFormatter::JsonEncodeKind UnifiedLogFormatter::needJsonEncode(const st
return JsonEncodeKind::DirectCopy;
}

void UnifiedLogFormatter::writeJSONString(FmtBuffer & buf, const std::string & str)
void writeJSONString(FmtBuffer & buf, const std::string & str)
{
buf.append('"');

Expand Down Expand Up @@ -226,4 +252,110 @@ void UnifiedLogFormatter::writeJSONString(FmtBuffer & buf, const std::string & s
buf.append('"');
}

void writeEscapedString(FmtBuffer & buf, const std::string & str)
{
auto encode_kind = needJsonEncode(str);
switch (encode_kind)
{
case JsonEncodeKind::DirectCopy:
buf.append(str);
break;
case JsonEncodeKind::AddQuoteAndCopy:
buf.append('"');
buf.append(str);
buf.append('"');
break;
case JsonEncodeKind::Encode:
writeJSONString(buf, str);
break;
}
}

template <bool enable_color>
void UnifiedLogFormatter<enable_color>::format(const Poco::Message & msg, std::string & text)
{
FmtBuffer buf;

// Timestamp
{
if constexpr (enable_color)
buf.append(BRIGHT_GREEN);

buf.append('[');
writeTimestamp(buf);
buf.append("] ");

if constexpr (enable_color)
buf.append(RESET_FONT);
}

// Priority
{
if constexpr (enable_color)
{
buf.append(BOLD_FONT);
writePriorityColor(buf, msg.getPriority());
}

buf.append('[');
writePriority(buf, msg.getPriority());
buf.append("] ");

if constexpr (enable_color)
buf.append(RESET_FONT);
}

// Source File
{
if constexpr (enable_color)
buf.append(MAGENTA);

if (unlikely(!msg.getSourceFile()))
buf.append("[<unknown>] ");
else
buf.fmtAppend(FMT_COMPILE("[{}:{}] "), msg.getSourceFile(), msg.getSourceLine());

if constexpr (enable_color)
buf.append(RESET_FONT);
}
// Message
{
if constexpr (enable_color)
writeMessageColor(buf, msg.getPriority());

buf.append('[');
writeEscapedString(buf, msg.getText());
buf.append("] ");

if constexpr (enable_color)
buf.append(RESET_FONT);
}

// Source and Identifiers
if constexpr (enable_color)
buf.append(BRIGHT_BLACK);

{
const std::string & source = msg.getSource();
if (!source.empty())
{
buf.append("[source=");
writeEscapedString(buf, source);
buf.append("] ");
}
}
// Thread ID
{
buf.fmtAppend(FMT_COMPILE("[thread_id={}]"), Poco::ThreadNumber::get());
}

if constexpr (enable_color)
buf.append(RESET_FONT);

text = buf.toString();
}

template class UnifiedLogFormatter<true>;
template class UnifiedLogFormatter<false>;

} // namespace DB
Loading

0 comments on commit e0c82c0

Please sign in to comment.