Skip to content

Commit

Permalink
config: bootstrap v1 JSON -> proto translation. (#1521)
Browse files Browse the repository at this point in the history
  • Loading branch information
htuch authored Aug 31, 2017
1 parent d123890 commit 34c0098
Show file tree
Hide file tree
Showing 77 changed files with 779 additions and 618 deletions.
1 change: 1 addition & 0 deletions include/envoy/server/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ envoy_cc_library(
"//include/envoy/access_log:access_log_interface",
"//include/envoy/http:filter_interface",
"//include/envoy/init:init_interface",
"//include/envoy/json:json_object_interface",
"//include/envoy/local_info:local_info_interface",
"//include/envoy/network:drain_decision_interface",
"//include/envoy/ratelimit:ratelimit_interface",
Expand Down
4 changes: 2 additions & 2 deletions include/envoy/server/configuration.h
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,9 @@ class Main {
virtual Optional<std::string> statsdTcpClusterName() PURE;

/**
* @return Optional<std::string> the optional UDP statsd address to write to.
* @return Network::Address::InstanceConstSharedPtr the optional UDP statsd address to write to.
*/
virtual Optional<std::string> statsdUdpIpAddress() PURE;
virtual Network::Address::InstanceConstSharedPtr statsdUdpIpAddress() PURE;

/**
* @return std::chrono::milliseconds the time interval between flushing to configured stat sinks.
Expand Down
8 changes: 0 additions & 8 deletions include/envoy/server/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,6 @@ class Options {
*/
virtual const std::string& configPath() PURE;

/**
* @return const std::string& the path to the v2 bootstrap file.
* TODO(htuch): We can eventually consolidate configPath()/bootstrapPath(), but today
* the config fetched from bootstrapPath() acts as an overlay to the config fetched from
* configPath() during v2 API bringup.
*/
virtual const std::string& bootstrapPath() PURE;

/**
* @return const std::string& the admin address output file.
*/
Expand Down
1 change: 0 additions & 1 deletion include/envoy/upstream/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ envoy_cc_library(
"//include/envoy/access_log:access_log_interface",
"//include/envoy/http:async_client_interface",
"//include/envoy/http:conn_pool_interface",
"//include/envoy/json:json_object_interface",
"//include/envoy/local_info:local_info_interface",
"//include/envoy/runtime:runtime_interface",
],
Expand Down
27 changes: 9 additions & 18 deletions include/envoy/upstream/cluster_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
#include "envoy/access_log/access_log.h"
#include "envoy/http/async_client.h"
#include "envoy/http/conn_pool.h"
#include "envoy/json/json_object.h"
#include "envoy/local_info/local_info.h"
#include "envoy/runtime/runtime.h"
#include "envoy/upstream/load_balancer.h"
Expand Down Expand Up @@ -120,14 +119,6 @@ class ClusterManager {

typedef std::unique_ptr<ClusterManager> ClusterManagerPtr;

/**
* Global configuration for any SDS clusters.
*/
struct SdsConfig {
std::string sds_cluster_name_;
std::chrono::milliseconds refresh_delay_;
};

/**
* Abstract interface for a CDS API provider.
*/
Expand Down Expand Up @@ -157,15 +148,14 @@ class ClusterManagerFactory {
virtual ~ClusterManagerFactory() {}

/**
* Allocate a cluster manager from configuration JSON.
* TODO(htuch): Once bootstrap is sufficiently capable, switch to a translation from the JSON v1
* cluster manager config -> v2 proto and drop the config parameter.
* Allocate a cluster manager from configuration proto.
*/
virtual ClusterManagerPtr
clusterManagerFromJson(const Json::Object& config, const envoy::api::v2::Bootstrap& bootstrap,
Stats::Store& stats, ThreadLocal::Instance& tls, Runtime::Loader& runtime,
Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info,
AccessLog::AccessLogManager& log_manager) PURE;
virtual ClusterManagerPtr clusterManagerFromProto(const envoy::api::v2::Bootstrap& bootstrap,
Stats::Store& stats, ThreadLocal::Instance& tls,
Runtime::Loader& runtime,
Runtime::RandomGenerator& random,
const LocalInfo::LocalInfo& local_info,
AccessLog::AccessLogManager& log_manager) PURE;

/**
* Allocate an HTTP connection pool.
Expand All @@ -186,7 +176,8 @@ class ClusterManagerFactory {
* Create a CDS API provider from configuration proto.
*/
virtual CdsApiPtr createCds(const envoy::api::v2::ConfigSource& cds_config,
const Optional<SdsConfig>& sds_config, ClusterManager& cm) PURE;
const Optional<envoy::api::v2::ConfigSource>& eds_config,
ClusterManager& cm) PURE;
};

} // namespace Upstream
Expand Down
2 changes: 1 addition & 1 deletion include/envoy/upstream/resource_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ namespace Upstream {
* arrays for each priority, but does not pollute the enum.
*/
enum class ResourcePriority { Default, High };
const size_t NumResourcePriorities = 2;
const std::size_t NumResourcePriorities = 2;

/**
* An individual resource tracked by the resource manager.
Expand Down
18 changes: 18 additions & 0 deletions source/common/config/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,24 @@ envoy_cc_library(
],
)

envoy_cc_library(
name = "bootstrap_json_lib",
srcs = ["bootstrap_json.cc"],
hdrs = ["bootstrap_json.h"],
external_deps = ["envoy_bootstrap"],
deps = [
":address_json_lib",
":cds_json_lib",
":json_utility_lib",
":lds_json_lib",
":utility_lib",
"//include/envoy/json:json_object_interface",
"//source/common/common:assert_lib",
"//source/common/json:config_schemas_lib",
"//source/common/protobuf:utility_lib",
],
)

envoy_cc_library(
name = "base_json_lib",
srcs = ["base_json.cc"],
Expand Down
131 changes: 131 additions & 0 deletions source/common/config/bootstrap_json.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
#include "common/config/bootstrap_json.h"

#include "common/common/assert.h"
#include "common/config/address_json.h"
#include "common/config/cds_json.h"
#include "common/config/json_utility.h"
#include "common/config/lds_json.h"
#include "common/config/utility.h"
#include "common/json/config_schemas.h"
#include "common/protobuf/utility.h"

namespace Envoy {
namespace Config {

void BootstrapJson::translateClusterManagerBootstrap(const Json::Object& json_cluster_manager,
envoy::api::v2::Bootstrap& bootstrap) {
json_cluster_manager.validateSchema(Json::Schema::CLUSTER_MANAGER_SCHEMA);

Optional<envoy::api::v2::ConfigSource> eds_config;
if (json_cluster_manager.hasObject("sds")) {
const auto json_sds = json_cluster_manager.getObject("sds");
auto* cluster = bootstrap.mutable_static_resources()->mutable_clusters()->Add();
Config::CdsJson::translateCluster(*json_sds->getObject("cluster"),
Optional<envoy::api::v2::ConfigSource>(), *cluster);
Config::Utility::translateEdsConfig(
*json_sds,
*bootstrap.mutable_dynamic_resources()->mutable_deprecated_v1()->mutable_sds_config());
eds_config.value(bootstrap.dynamic_resources().deprecated_v1().sds_config());
}

if (json_cluster_manager.hasObject("cds")) {
const auto json_cds = json_cluster_manager.getObject("cds");
auto* cluster = bootstrap.mutable_static_resources()->mutable_clusters()->Add();
Config::CdsJson::translateCluster(*json_cds->getObject("cluster"), eds_config, *cluster);
Config::Utility::translateCdsConfig(
*json_cds, *bootstrap.mutable_dynamic_resources()->mutable_cds_config());
}

for (const Json::ObjectSharedPtr& json_cluster :
json_cluster_manager.getObjectArray("clusters")) {
auto* cluster = bootstrap.mutable_static_resources()->mutable_clusters()->Add();
Config::CdsJson::translateCluster(*json_cluster, eds_config, *cluster);
}

auto* cluster_manager = bootstrap.mutable_cluster_manager();
JSON_UTIL_SET_STRING(json_cluster_manager, *cluster_manager, local_cluster_name);
if (json_cluster_manager.hasObject("outlier_detection")) {
JSON_UTIL_SET_STRING(*json_cluster_manager.getObject("outlier_detection"),
*cluster_manager->mutable_outlier_detection(), event_log_path);
}
}

void BootstrapJson::translateBootstrap(const Json::Object& json_config,
envoy::api::v2::Bootstrap& bootstrap) {
json_config.validateSchema(Json::Schema::TOP_LEVEL_CONFIG_SCHEMA);

translateClusterManagerBootstrap(*json_config.getObject("cluster_manager"), bootstrap);

if (json_config.hasObject("lds")) {
auto* lds_config = bootstrap.mutable_dynamic_resources()->mutable_lds_config();
Config::Utility::translateLdsConfig(*json_config.getObject("lds"), *lds_config);
}

for (const auto json_listener : json_config.getObjectArray("listeners")) {
auto* listener = bootstrap.mutable_static_resources()->mutable_listeners()->Add();
Config::LdsJson::translateListener(*json_listener, *listener);
}

JSON_UTIL_SET_STRING(json_config, bootstrap, flags_path);

auto* stats_sinks = bootstrap.mutable_stats_sinks();
if (json_config.hasObject("statsd_udp_ip_address")) {
auto* stats_sink = stats_sinks->Add();
stats_sink->set_name("envoy.statsd");
envoy::api::v2::StatsdSink statsd_sink;
AddressJson::translateAddress(json_config.getString("statsd_udp_ip_address"), false, true,
*statsd_sink.mutable_address());
MessageUtil::jsonConvert(statsd_sink, *stats_sink->mutable_config());
}

if (json_config.hasObject("statsd_tcp_cluster_name")) {
auto* stats_sink = stats_sinks->Add();
stats_sink->set_name("envoy.statsd");
envoy::api::v2::StatsdSink statsd_sink;
statsd_sink.set_tcp_cluster_name(json_config.getString("statsd_tcp_cluster_name"));
MessageUtil::jsonConvert(statsd_sink, *stats_sink->mutable_config());
}

JSON_UTIL_SET_DURATION(json_config, bootstrap, stats_flush_interval);

auto* watchdog = bootstrap.mutable_watchdog();
JSON_UTIL_SET_DURATION(json_config, *watchdog, miss_timeout);
JSON_UTIL_SET_DURATION(json_config, *watchdog, megamiss_timeout);
JSON_UTIL_SET_DURATION(json_config, *watchdog, kill_timeout);
JSON_UTIL_SET_DURATION(json_config, *watchdog, multikill_timeout);

const auto http = json_config.getObject("tracing", true)->getObject("http", true);
if (http->hasObject("driver")) {
const auto driver = http->getObject("driver");
auto* http_tracing = bootstrap.mutable_tracing()->mutable_http();
http_tracing->set_name("envoy." + driver->getString("type"));
MessageUtil::loadFromJson(driver->getObject("config")->asJsonString(),
*http_tracing->mutable_config());
}

if (json_config.hasObject("rate_limit_service")) {
const auto json_rate_limit_service = json_config.getObject("rate_limit_service");
auto* rate_limit_service = bootstrap.mutable_rate_limit_service();
ASSERT(json_rate_limit_service->getString("type") == "grpc_service");
JSON_UTIL_SET_STRING(*json_rate_limit_service->getObject("config"), *rate_limit_service,
cluster_name);
}

const auto json_admin = json_config.getObject("admin");
auto* admin = bootstrap.mutable_admin();
JSON_UTIL_SET_STRING(*json_admin, *admin, access_log_path);
JSON_UTIL_SET_STRING(*json_admin, *admin, profile_path);
AddressJson::translateAddress(json_admin->getString("address"), true, true,
*admin->mutable_address());

if (json_config.hasObject("runtime")) {
const auto json_runtime = json_config.getObject("runtime");
auto* runtime = bootstrap.mutable_runtime();
JSON_UTIL_SET_STRING(*json_runtime, *runtime, symlink_root);
JSON_UTIL_SET_STRING(*json_runtime, *runtime, subdirectory);
JSON_UTIL_SET_STRING(*json_runtime, *runtime, override_subdirectory);
}
}

} // namespace Config
} // namespace Envoy
30 changes: 30 additions & 0 deletions source/common/config/bootstrap_json.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#pragma once

#include "envoy/json/json_object.h"

#include "api/bootstrap.pb.h"

namespace Envoy {
namespace Config {

class BootstrapJson {
public:
/**
* Translate a v1 JSON cluster manager object to v2 envoy::api::v2::Bootstrap.
* @param json_cluster_manager source v1 JSON cluster manager object.
* @param bootstrap destination v2 envoy::api::v2::Bootstrap.
*/
static void translateClusterManagerBootstrap(const Json::Object& json_cluster_manager,
envoy::api::v2::Bootstrap& bootstrap);

/**
* Translate a v1 JSON static config object to v2 envoy::api::v2::Bootstrap.
* @param json_config source v1 JSON static config object.
* @param bootstrap destination v2 envoy::api::v2::Bootstrap.
*/
static void translateBootstrap(const Json::Object& json_config,
envoy::api::v2::Bootstrap& bootstrap);
};

} // namespace Config
} // namespace Envoy
7 changes: 3 additions & 4 deletions source/common/config/cds_json.cc
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ void CdsJson::translateOutlierDetection(
}

void CdsJson::translateCluster(const Json::Object& json_cluster,
const Optional<Upstream::SdsConfig>& sds_config,
const Optional<envoy::api::v2::ConfigSource>& eds_config,
envoy::api::v2::Cluster& cluster) {
json_cluster.validateSchema(Json::Schema::CLUSTER_SCHEMA);
cluster.set_name(json_cluster.getString("name"));
Expand Down Expand Up @@ -120,9 +120,8 @@ void CdsJson::translateCluster(const Json::Object& json_cluster,
} else {
ASSERT(string_type == "sds");
cluster.set_type(envoy::api::v2::Cluster::EDS);
auto* eds_cluster_config = cluster.mutable_eds_cluster_config();
Utility::sdsConfigToEdsConfig(sds_config.value(), *eds_cluster_config->mutable_eds_config());
eds_cluster_config->set_service_name(json_cluster.getString("service_name", ""));
cluster.mutable_eds_cluster_config()->mutable_eds_config()->CopyFrom(eds_config.value());
JSON_UTIL_SET_STRING(json_cluster, *cluster.mutable_eds_cluster_config(), service_name);
}

JSON_UTIL_SET_DURATION(json_cluster, cluster, cleanup_interval);
Expand Down
4 changes: 2 additions & 2 deletions source/common/config/cds_json.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ class CdsJson {
/**
* Translate a v1 JSON Cluster to v2 envoy::api::v2::Cluster.
* @param json_cluster source v1 JSON Cluster object.
* @param sds_config SDS config if 'sds' discovery type.
* @param eds_config SDS config if 'sds' discovery type.
* @param cluster destination v2 envoy::api::v2::Cluster.
*/
static void translateCluster(const Json::Object& json_cluster,
const Optional<Upstream::SdsConfig>& sds_config,
const Optional<envoy::api::v2::ConfigSource>& eds_config,
envoy::api::v2::Cluster& cluster);
};

Expand Down
12 changes: 5 additions & 7 deletions source/common/config/utility.cc
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,11 @@ Utility::apiConfigSourceRefreshDelay(const envoy::api::v2::ApiConfigSource& api_
Protobuf::util::TimeUtil::DurationToMilliseconds(api_config_source.refresh_delay()));
}

void Utility::sdsConfigToEdsConfig(const Upstream::SdsConfig& sds_config,
envoy::api::v2::ConfigSource& eds_config) {
auto* api_config_source = eds_config.mutable_api_config_source();
api_config_source->set_api_type(envoy::api::v2::ApiConfigSource::REST_LEGACY);
api_config_source->add_cluster_name(sds_config.sds_cluster_name_);
api_config_source->mutable_refresh_delay()->CopyFrom(
Protobuf::util::TimeUtil::MillisecondsToDuration(sds_config.refresh_delay_.count()));
void Utility::translateEdsConfig(const Json::Object& json_config,
envoy::api::v2::ConfigSource& eds_config) {
translateApiConfigSource(json_config.getObject("cluster")->getString("name"),
json_config.getInteger("refresh_delay_ms", 30000),
*eds_config.mutable_api_config_source());
}

void Utility::translateCdsConfig(const Json::Object& json_config,
Expand Down
9 changes: 5 additions & 4 deletions source/common/config/utility.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#pragma once

#include "envoy/config/subscription.h"
#include "envoy/json/json_object.h"
#include "envoy/local_info/local_info.h"
#include "envoy/upstream/cluster_manager.h"

Expand Down Expand Up @@ -69,12 +70,12 @@ class Utility {
const LocalInfo::LocalInfo& local_info);

/**
* Convert a v1 SdsConfig to v2 EDS envoy::api::v2::ConfigSource.
* @param sds_config source v1 SdsConfig.
* Convert a v1 SDS JSON config to v2 EDS envoy::api::v2::ConfigSource.
* @param json_config source v1 SDS JSON config.
* @param eds_config destination v2 EDS envoy::api::v2::ConfigSource.
*/
static void sdsConfigToEdsConfig(const Upstream::SdsConfig& sds_config,
envoy::api::v2::ConfigSource& eds_config);
static void translateEdsConfig(const Json::Object& json_config,
envoy::api::v2::ConfigSource& eds_config);

/**
* Convert a v1 CDS JSON config to v2 CDS envoy::api::v2::ConfigSource.
Expand Down
2 changes: 1 addition & 1 deletion source/common/network/utility.cc
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ Address::InstanceConstSharedPtr Utility::fromProtoAddress(const envoy::api::v2::
case envoy::api::v2::Address::kPipe:
return Address::InstanceConstSharedPtr{new Address::PipeInstance(address.pipe().path())};
default:
NOT_REACHED;
throw EnvoyException("Address must be a socket or pipe: " + address.DebugString());
}
}

Expand Down
10 changes: 10 additions & 0 deletions source/common/protobuf/utility.cc
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,14 @@ std::string MessageUtil::getJsonStringFromMessage(const Protobuf::Message& messa
return json;
}

void MessageUtil::jsonConvert(const Protobuf::Message& source, Protobuf::Message& dest) {
// TODO(htuch): Consolidate with the inflight cleanups here.
Protobuf::util::JsonOptions json_options;
ProtobufTypes::String json;
const auto status = Protobuf::util::MessageToJsonString(source, &json, json_options);
// This should always succeed unless something crash-worthy such as out-of-memory.
RELEASE_ASSERT(status.ok());
MessageUtil::loadFromJson(json, dest);
}

} // namespace Envoy
Loading

0 comments on commit 34c0098

Please sign in to comment.