From e3eb7c81e369a70642a16ca36e6576201aa90740 Mon Sep 17 00:00:00 2001 From: Limin Wang Date: Tue, 10 Jan 2017 15:34:32 -0800 Subject: [PATCH 01/15] Created check security rules file and a few dummy/helper functions. (#40) * Created check security rules file and a few dummy/helper functions. And added it to check work flow. * Fix format. --- contrib/endpoints/src/api_manager/BUILD | 2 + .../src/api_manager/check_security_rules.cc | 106 ++++++++++++++++++ .../src/api_manager/check_security_rules.h | 32 ++++++ .../src/api_manager/check_workflow.cc | 3 + 4 files changed, 143 insertions(+) create mode 100644 contrib/endpoints/src/api_manager/check_security_rules.cc create mode 100644 contrib/endpoints/src/api_manager/check_security_rules.h diff --git a/contrib/endpoints/src/api_manager/BUILD b/contrib/endpoints/src/api_manager/BUILD index e3d54b46eac..38671d6e90e 100644 --- a/contrib/endpoints/src/api_manager/BUILD +++ b/contrib/endpoints/src/api_manager/BUILD @@ -68,6 +68,8 @@ cc_library( "api_manager_impl.cc", "check_auth.cc", "check_auth.h", + "check_security_rules.cc", + "check_security_rules.h", "check_service_control.cc", "check_service_control.h", "check_workflow.cc", diff --git a/contrib/endpoints/src/api_manager/check_security_rules.cc b/contrib/endpoints/src/api_manager/check_security_rules.cc new file mode 100644 index 00000000000..b1d6b90ac36 --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_security_rules.cc @@ -0,0 +1,106 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +#include "contrib/endpoints/src/api_manager/check_security_rules.h" + +#include + +#include "contrib/endpoints/include/api_manager/api_manager.h" +#include "contrib/endpoints/include/api_manager/request.h" + +using ::google::api_manager::utils::Status; + +namespace google { +namespace api_manager { + +namespace { + +const char kFirebaseServerStaging[] = + "https://staging-firebaserules.sandbox.googleapis.com/"; + +// An AuthzChecker object is created for every incoming request. It does +// authorizaiton by calling Firebase Rules service. +class AuthzChecker : public std::enable_shared_from_this { + public: + AuthzChecker(std::shared_ptr context, + std::function continuation); + + void Check(); + + private: + // Helper function to send a http GET request. + void HttpFetch(const std::string &url, const std::string &request_body, + std::function continuation); + + // Get Auth token for accessing Firebase Rules service. + const std::string &GetAuthToken(); + + // Request context. + std::shared_ptr context_; + + // Pointer to access ESP running environment. + ApiManagerEnvInterface *env_; + + // The final continuation function. + std::function on_done_; +}; + +AuthzChecker::AuthzChecker(std::shared_ptr context, + std::function continuation) + : context_(context), + env_(context_->service_context()->env()), + on_done_(continuation) {} + +void AuthzChecker::Check() { + // TODO: Check service config to see if "useSecurityRules" is specified. + // If so, call Firebase Rules service TestRuleset API. +} + +const std::string &AuthzChecker::GetAuthToken() { + // TODO: Get Auth token for accessing Firebase Rules service. + static std::string empty; + return empty; +} + +void AuthzChecker::HttpFetch( + const std::string &url, const std::string &request_body, + std::function continuation) { + std::unique_ptr request(new HTTPRequest([continuation]( + Status status, std::map &&, + std::string &&body) { continuation(status, std::move(body)); })); + if (!request) { + continuation(Status(Code::INTERNAL, "Out of memory"), ""); + return; + } + + request->set_method("POST") + .set_url(url) + .set_auth_token(GetAuthToken()) + .set_header("Content-Type", "application/json") + .set_body(request_body); + env_->RunHTTPRequest(std::move(request)); +} + +} // namespace + +void CheckSecurityRules(std::shared_ptr context, + std::function continuation) { + std::shared_ptr authzChecker = + std::make_shared(context, continuation); + authzChecker->Check(); +} + +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/check_security_rules.h b/contrib/endpoints/src/api_manager/check_security_rules.h new file mode 100644 index 00000000000..bc971c48786 --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_security_rules.h @@ -0,0 +1,32 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_MANAGER_CHECK_SECURITY_RULES_H_ +#define API_MANAGER_CHECK_SECURITY_RULES_H_ + +#include "contrib/endpoints/include/api_manager/utils/status.h" +#include "contrib/endpoints/src/api_manager/context/request_context.h" + +namespace google { +namespace api_manager { + +// This function checks security rules for a given request. +// It is called by CheckWorkflow class when processing a request. +void CheckSecurityRules(std::shared_ptr context, + std::function continuation); + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_CHECK_SECURITY_RULES_H_ diff --git a/contrib/endpoints/src/api_manager/check_workflow.cc b/contrib/endpoints/src/api_manager/check_workflow.cc index 8335d779142..7c869ab30cc 100644 --- a/contrib/endpoints/src/api_manager/check_workflow.cc +++ b/contrib/endpoints/src/api_manager/check_workflow.cc @@ -16,6 +16,7 @@ #include "contrib/endpoints/src/api_manager/check_workflow.h" #include "contrib/endpoints/src/api_manager/check_auth.h" +#include "contrib/endpoints/src/api_manager/check_security_rules.h" #include "contrib/endpoints/src/api_manager/check_service_control.h" #include "contrib/endpoints/src/api_manager/fetch_metadata.h" @@ -33,6 +34,8 @@ void CheckWorkflow::RegisterAll() { Register(CheckAuth); // Checks service control. Register(CheckServiceControl); + // Check Security Rules. + Register(CheckSecurityRules); } void CheckWorkflow::Register(CheckHandler handler) { From fe30dc1c237ce4d599baaac93755b0d0787a67dd Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Tue, 24 Jan 2017 09:25:47 -0800 Subject: [PATCH 02/15] Firebase: Merge from master. (#53) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect (#38) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect * Fixed style. * Rebase Envoy (#41) * Update prototype to use iptables (#42) * Rebase to fixed Envoy (#43) * Handle HEAD request. (#34) * Handle HEAD request. * Try with GET if HEAD fails. * Address comments. * Format file. * Expose bazel target (#48) * Try again (#49) --- .../api_manager/context/service_context.cc | 13 +- contrib/tools/server/Makefile | 6 + contrib/tools/server/README.md | 25 +++ contrib/tools/server/server.c | 169 ++++++++++++++++++ src/envoy/prototype/BUILD | 2 + src/envoy/prototype/README.md | 13 ++ src/envoy/prototype/envoy-esp.conf | 11 +- src/envoy/repositories.bzl | 2 +- test/backend/echo/echo.go | 21 +-- 9 files changed, 244 insertions(+), 18 deletions(-) create mode 100644 contrib/tools/server/Makefile create mode 100644 contrib/tools/server/README.md create mode 100644 contrib/tools/server/server.c diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index b9f753f0625..d8fc9dfc409 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -41,6 +41,9 @@ const double kDefaultTraceSampleQps = 0.1; // The time window to send intermediate report for Grpc streaming (second). // Default to 10s. const int kIntermediateReportInterval = 10; + +const char kHTTPHeadMethod[] = "HEAD"; +const char kHTTPGetMethod[] = "GET"; } ServiceContext::ServiceContext(std::unique_ptr env, @@ -74,7 +77,15 @@ MethodCallInfo ServiceContext::GetMethodCallInfo( if (config_ == nullptr) { return MethodCallInfo(); } - return config_->GetMethodCallInfo(http_method, url, query_params); + MethodCallInfo method_call_info = + config_->GetMethodCallInfo(http_method, url, query_params); + // HEAD should be treated as GET unless it is specified from service_config. + if (method_call_info.method_info == nullptr && + http_method == kHTTPHeadMethod) { + method_call_info = + config_->GetMethodCallInfo(kHTTPGetMethod, url, query_params); + } + return method_call_info; } const std::string& ServiceContext::project_id() const { diff --git a/contrib/tools/server/Makefile b/contrib/tools/server/Makefile new file mode 100644 index 00000000000..d7c7d758686 --- /dev/null +++ b/contrib/tools/server/Makefile @@ -0,0 +1,6 @@ +server: + $(CC) -g -o server server.c + +clean: + rm server + diff --git a/contrib/tools/server/README.md b/contrib/tools/server/README.md new file mode 100644 index 00000000000..f962c2c640b --- /dev/null +++ b/contrib/tools/server/README.md @@ -0,0 +1,25 @@ +# Sample TCP Server + +This is a simple TCP server that listens on the specified port (3490 by default) and replies to incoming connections by providing info about src/destination ip/port. + +It demonstrates the use of the getsockopt() system call with the ORIGINAL_DST option to retrieve the original destination ip/port after an iptables redirect. + +So, for example, if you have the server listening on port 3490 on the local machine and an iptables rule like: + +``` +iptables -t nat -I OUTPUT 1 -p tcp --dport 4000:5000 -j REDIRECT --to-port 3490 +``` +your will see: + +``` +$ telnet localhost 3490 +FROM 127.0.0.1:44978, TO 127.0.0.1:3490, ORIG DEST 127.0.0.1:3490 + +$ telnet localhost 4100 +FROM 127.0.0.1:35476, TO 127.0.0.1:3490, ORIG DEST 127.0.0.1:4100 + +$ telnet 1.1.1.1 5000 +FROM 100.100.100.100:60275, TO 127.0.0.1:3490, ORIG DEST 1.1.1.1:5000 +``` + + diff --git a/contrib/tools/server/server.c b/contrib/tools/server/server.c new file mode 100644 index 00000000000..77a2d97e004 --- /dev/null +++ b/contrib/tools/server/server.c @@ -0,0 +1,169 @@ +/* +** server.c + * Demo server to check that we can extract all parameters from an incoming +*request + * and respond to it +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFAULT_PORT \ + "3490" // the port users will be connecting to, if not specified on command + // line + +#define QUEUE_SIZE 10 // pending connections queue size + +void sigchld_handler(int s) { + int saved_errno = errno; + + while (waitpid(-1, NULL, WNOHANG) > 0) + ; + + errno = saved_errno; +} + +int main(int argc, char *argv[]) { + struct sigaction sa; + const int yes = 1; + char *port = DEFAULT_PORT; + int rv; + + if (argc > 1) { + port = argv[1]; + } + + struct addrinfo hints; + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; // use my IP + + struct addrinfo *server_info = NULL; + if ((rv = getaddrinfo(NULL, port, &hints, &server_info)) != 0) { + fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv)); + return 1; + } + + struct addrinfo *p; + int sockfd; // listen on sock_fd + for (p = server_info; p != NULL; p = p->ai_next) { + if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol)) == -1) { + perror("server: socket"); + continue; + } + + if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) { + perror("setsockopt"); + exit(1); + } + + if (bind(sockfd, p->ai_addr, p->ai_addrlen) == -1) { + close(sockfd); + perror("server: bind"); + continue; + } + + break; + } + + freeaddrinfo(server_info); + + if (p == NULL) { + fprintf(stderr, "server: failed to bind\n"); + exit(1); + } + if (listen(sockfd, QUEUE_SIZE) == -1) { + perror("listen"); + exit(1); + } + + sa.sa_handler = sigchld_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + if (sigaction(SIGCHLD, &sa, NULL) == -1) { + perror("sigaction"); + exit(1); + } + + char bind_addr_str[INET6_ADDRSTRLEN] = {0}; + struct sockaddr_in *bind_sock_addr = (struct sockaddr_in *)p->ai_addr; + inet_ntop(p->ai_family, &bind_sock_addr->sin_addr, bind_addr_str, + p->ai_addrlen); + printf("server %s: waiting for connections on port %s:%u...\n", port, + bind_addr_str, ntohs(bind_sock_addr->sin_port)); + + while (1) { + socklen_t addr_len = sizeof(struct sockaddr_storage); + int addr_str_len = INET6_ADDRSTRLEN; + + struct sockaddr_storage their_addr = { + 0}; // connector's address information + int accepted_fd = accept(sockfd, (struct sockaddr *)&their_addr, &addr_len); + if (accepted_fd == -1) { + perror("accept"); + continue; + } + + struct sockaddr_in *src_sock_addr = (struct sockaddr_in *)&their_addr; + char their_addr_str[INET6_ADDRSTRLEN] = {0}; + inet_ntop(their_addr.ss_family, &src_sock_addr->sin_addr, their_addr_str, + addr_str_len); + printf("server %s: got connection FROM %s:%u\n", port, their_addr_str, + ntohs(src_sock_addr->sin_port)); + + struct sockaddr_storage my_addr = {0}; // my address information + struct sockaddr_in *dst_sock_addr = (struct sockaddr_in *)&my_addr; + char my_addr_str[INET6_ADDRSTRLEN] = {0}; + getsockname(accepted_fd, (struct sockaddr *)dst_sock_addr, &addr_len); + inet_ntop(my_addr.ss_family, &dst_sock_addr->sin_addr, my_addr_str, + addr_str_len); + printf("server %s: got connection TO %s:%u\n", port, my_addr_str, + ntohs(dst_sock_addr->sin_port)); + + struct sockaddr_storage orig_addr = {0}; // orig address information + struct sockaddr_in *orig_sock_addr = (struct sockaddr_in *)&orig_addr; + char orig_addr_str[INET6_ADDRSTRLEN] = {0}; + int status = getsockopt(accepted_fd, SOL_IP, SO_ORIGINAL_DST, + orig_sock_addr, &addr_len); + + if (status == 0) { + inet_ntop(orig_addr.ss_family, &orig_sock_addr->sin_addr, orig_addr_str, + addr_str_len); + printf("server %s: ORIG DEST %s:%u\n", port, orig_addr_str, + ntohs(orig_sock_addr->sin_port)); + } else { + printf("Could not get orig destination from accepted socket.\n"); + } + + if (!fork()) { // this is the child process + + close(sockfd); + char msg[256] = {0}; + snprintf(msg, 256, "FROM %s:%u, TO %s:%u, ORIG DEST %s:%u\n", + their_addr_str, ntohs(src_sock_addr->sin_port), my_addr_str, + ntohs(dst_sock_addr->sin_port), orig_addr_str, + ntohs(orig_sock_addr->sin_port)); + + if (send(accepted_fd, msg, strlen(msg), 0) == -1) { + perror("send"); + } + close(accepted_fd); + exit(0); + } + close(accepted_fd); + } + + return 0; +} diff --git a/src/envoy/prototype/BUILD b/src/envoy/prototype/BUILD index e5cbaeba10c..258f2046cd7 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/prototype/BUILD @@ -15,6 +15,8 @@ ################################################################################ # +package(default_visibility = ["//visibility:public"]) + cc_binary( name = "envoy_esp", srcs = [ diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md index 2ddbe98459c..00e73a169d6 100644 --- a/src/envoy/prototype/README.md +++ b/src/envoy/prototype/README.md @@ -36,6 +36,19 @@ This Proxy will use Envoy and talk to Mixer server. go run echo.go ``` +* Modify your iptables: + +``` + sudo iptables -t nat -A OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 +``` + +Once you are done, you should remove this rule: + +``` + sudo iptables -t nat -D OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 +``` + + * Start Envoy proxy, run ``` diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf index dff3bd67b1a..11690bed565 100644 --- a/src/envoy/prototype/envoy-esp.conf +++ b/src/envoy/prototype/envoy-esp.conf @@ -1,7 +1,14 @@ { "listeners": [ + { + "port": 9092, + "bind_to_port": true, + "use_original_dst": true, + "filters": [] + }, { "port": 9090, + "bind_to_port": false, "filters": [ { "type": "read", @@ -26,7 +33,7 @@ }, "access_log": [ { - "path": "/tmp/access.envoy" + "path": "/dev/stdout" } ], "filters": [ @@ -50,7 +57,7 @@ } ], "admin": { - "access_log_path": "/tmp/access.envoy", + "access_log_path": "/dev/stdout", "port": 9001 }, "cluster_manager": { diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 56fabce07ac..dd36a23d456 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "6b1336a786ebe56c45a1a349ddf706e0526c1ec1", # 2017-01-03 + commit = "c36bb76e304ab4c5584a07303e2953a170888319", build_file_content = BUILD, ) diff --git a/test/backend/echo/echo.go b/test/backend/echo/echo.go index 51aaa07e353..ef13f515ff1 100644 --- a/test/backend/echo/echo.go +++ b/test/backend/echo/echo.go @@ -23,19 +23,22 @@ import ( "io/ioutil" "net/http" "strconv" - "sync" - "time" ) var ( port = flag.Int("port", 8080, "default http port") - mu sync.Mutex requests = 0 data = 0 ) func handler(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%v %v %v %v\n", r.Method, r.URL, r.Proto, r.RemoteAddr) + for name, headers := range r.Header { + for _, h := range headers { + fmt.Printf("%v: %v\n", name, h) + } + } body, err := ioutil.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -51,24 +54,14 @@ func handler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(body) - mu.Lock() requests++ data += len(body) - defer mu.Unlock() + fmt.Printf("Requests Requests: %v Data: %v\n", requests, data) } func main() { flag.Parse() - go func() { - for { - mu.Lock() - fmt.Printf("Requests Requests: %v Data: %v\n", requests, data) - mu.Unlock() - time.Sleep(time.Second) - } - }() - fmt.Printf("Listening on port %v\n", *port) http.HandleFunc("/", handler) From df8c69c2c062abadfc880c09e5514f427577a182 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Thu, 2 Feb 2017 15:31:56 -0800 Subject: [PATCH 03/15] Enable ESP to invoke Firebase Security rules. (#54) * Enable ESP to invoke Firebase Security rules. * Address code review comments. * Remove some debug logs * Add proto file to capture TestRulesetRequest. * clang-format files * Resolve a merge issue with previous commit * Allow security rules to disabled via serverconfig * format file * Addressed Wayne's review comments. * Add firebase server to Server Config. * Address Lizan's review comments * Address review comments. * Disable check rules service by default. * Address more review comments. * Fix a check. * Delete unwanted constant. * Address Wayne's comments and add a simple config test. * Address a review comment. * Add negative test case for config * Address code review * Remove unwanted const std::string --- contrib/endpoints/src/api_manager/BUILD | 14 + contrib/endpoints/src/api_manager/auth.h | 2 + .../auth/lib/auth_jwt_validator.cc | 9 +- .../api_manager/auth/service_account_token.h | 1 + .../endpoints/src/api_manager/check_auth.cc | 2 + .../src/api_manager/check_security_rules.cc | 331 +++++++++++++++--- contrib/endpoints/src/api_manager/config.cc | 8 + contrib/endpoints/src/api_manager/config.h | 3 + .../endpoints/src/api_manager/config_test.cc | 39 +++ .../src/api_manager/context/request_context.h | 11 +- .../api_manager/context/service_context.cc | 17 +- .../src/api_manager/context/service_context.h | 5 + .../api_manager/proto/security_rules.proto | 57 +++ .../src/api_manager/proto/server_config.proto | 9 + 14 files changed, 462 insertions(+), 46 deletions(-) create mode 100644 contrib/endpoints/src/api_manager/proto/security_rules.proto diff --git a/contrib/endpoints/src/api_manager/BUILD b/contrib/endpoints/src/api_manager/BUILD index 38671d6e90e..ffe4f060b7c 100644 --- a/contrib/endpoints/src/api_manager/BUILD +++ b/contrib/endpoints/src/api_manager/BUILD @@ -38,6 +38,19 @@ cc_proto_library( visibility = ["//visibility:public"], ) +cc_proto_library( + name = "security_rules_proto", + srcs = [ + "proto/security_rules.proto", + ], + default_runtime = "//external:protobuf", + protoc = "//external:protoc", + visibility = ["//visibility:public"], + deps = [ + "//external:cc_wkt_protos", + ], +) + cc_library( name = "auth_headers", hdrs = [ @@ -99,6 +112,7 @@ cc_library( ":auth_headers", ":impl_headers", ":server_config_proto", + ":security_rules_proto", "//contrib/endpoints/src/api_manager/auth", "//contrib/endpoints/src/api_manager/cloud_trace", "//contrib/endpoints/src/api_manager/context", diff --git a/contrib/endpoints/src/api_manager/auth.h b/contrib/endpoints/src/api_manager/auth.h index aff735bc1aa..e69718e95f6 100644 --- a/contrib/endpoints/src/api_manager/auth.h +++ b/contrib/endpoints/src/api_manager/auth.h @@ -40,6 +40,8 @@ struct UserInfo { // Authorized party of the incoming JWT. // See http://openid.net/specs/openid-connect-core-1_0.html#IDToken std::string authorized_party; + // String of claims + std::string claims; // Returns audiences as a comma separated strings. std::string AudiencesAsString() const { diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc index 8da6ef2166c..2df9a843b71 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc @@ -699,12 +699,19 @@ grpc_jwt_verifier_status JwtValidatorImpl::FillUserInfoAndSetExp( // Optional field. const grpc_json *grpc_json = grpc_jwt_claims_json(claims_); + + char *json_str = + grpc_json_dump_to_string(const_cast<::grpc_json *>(grpc_json), 0); + if (json_str != nullptr) { + user_info->claims = json_str; + gpr_free(json_str); + } + const char *email = GetStringValue(grpc_json, "email"); user_info->email = email == nullptr ? "" : email; const char *authorized_party = GetStringValue(grpc_json, "azp"); user_info->authorized_party = authorized_party == nullptr ? "" : authorized_party; - exp_ = system_clock::from_time_t(grpc_jwt_claims_expires_at(claims_).tv_sec); return GRPC_JWT_VERIFIER_OK; diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token.h b/contrib/endpoints/src/api_manager/auth/service_account_token.h index 211377449f4..51e99c65b89 100644 --- a/contrib/endpoints/src/api_manager/auth/service_account_token.h +++ b/contrib/endpoints/src/api_manager/auth/service_account_token.h @@ -64,6 +64,7 @@ class ServiceAccountToken { enum JWT_TOKEN_TYPE { JWT_TOKEN_FOR_SERVICE_CONTROL = 0, JWT_TOKEN_FOR_CLOUD_TRACING, + JWT_TOKEN_FOR_FIREBASE, JWT_TOKEN_TYPE_MAX, }; // Set audience. Only calcualtes JWT token with specified audience. diff --git a/contrib/endpoints/src/api_manager/check_auth.cc b/contrib/endpoints/src/api_manager/check_auth.cc index ce9a2c159ff..b4aae8bb2a1 100644 --- a/contrib/endpoints/src/api_manager/check_auth.cc +++ b/contrib/endpoints/src/api_manager/check_auth.cc @@ -243,6 +243,8 @@ void AuthChecker::CheckAudience(bool cache_hit) { context_->set_auth_audience(audience); context_->set_auth_authorized_party(user_info_.authorized_party); + context_->set_auth_claims(user_info_.claims); + // Remove http/s header and trailing '/' for issuer. std::string issuer = utils::GetUrlContent(user_info_.issuer); if (!context_->method()->isIssuerAllowed(issuer)) { diff --git a/contrib/endpoints/src/api_manager/check_security_rules.cc b/contrib/endpoints/src/api_manager/check_security_rules.cc index b1d6b90ac36..45fa30ae7bd 100644 --- a/contrib/endpoints/src/api_manager/check_security_rules.cc +++ b/contrib/endpoints/src/api_manager/check_security_rules.cc @@ -14,82 +14,332 @@ // //////////////////////////////////////////////////////////////////////////////// #include "contrib/endpoints/src/api_manager/check_security_rules.h" +#include +#include +#include "contrib/endpoints/src/api_manager/auth/lib/json_util.h" +#include "contrib/endpoints/src/api_manager/proto/security_rules.pb.h" +#include "contrib/endpoints/src/api_manager/utils/marshalling.h" -#include - -#include "contrib/endpoints/include/api_manager/api_manager.h" -#include "contrib/endpoints/include/api_manager/request.h" - +using ::google::api_manager::auth::GetProperty; +using ::google::api_manager::auth::GetStringValue; using ::google::api_manager::utils::Status; +using ::google::protobuf::Map; +using ::google::protobuf::util::error::Code; namespace google { namespace api_manager { - namespace { -const char kFirebaseServerStaging[] = - "https://staging-firebaserules.sandbox.googleapis.com/"; +const char kFailedFirebaseReleaseFetch[] = "Failed to fetch Firebase Release"; +const char kFailedFirebaseTest[] = "Failed to execute Firebase Test"; +const char kInvalidResponse[] = "Invalid JSON response from Firebase Service"; +const char kTestSuccess[] = "SUCCESS"; +const char kHttpGetMethod[] = "GET"; +const char kHttpPostMethod[] = "POST"; +const char kHttpHeadMethod[] = "HEAD"; +const char kHttpOptionsMethod[] = "OPTIONS"; +const char kHttpDeleteMethod[] = "DELETE"; +const char kFirebaseCreateMethod[] = "create"; +const char kFirebaseGetMethod[] = "get"; +const char kFirebaseDeleteMethod[] = "delete"; +const char kFirebaseUpdateMethod[] = "update"; +const char kV1[] = "v1/"; +const char kTestQuery[] = ":test?alt=json"; +const char kProjects[] = "projects/"; +const char kReleases[] = "/releases/"; +const char kRulesetName[] = "rulesetName"; +const char kTestResults[] = "testResults"; +const char kState[] = "state"; +const char kToken[] = "token"; +const char kAuth[] = "auth"; +const char kRequest[] = "request"; +const char kContentType[] = "Content-Type"; +const char kApplication[] = "application/json"; + +void SetProtoValue(const std::string &key, + const ::google::protobuf::Value &value, + ::google::protobuf::Value *head) { + ::google::protobuf::Struct *s = head->mutable_struct_value(); + Map *fields = s->mutable_fields(); + (*fields)[key] = value; +} + +std::string GetReleaseName(const context::RequestContext &context) { + return context.service_context()->service_name() + ":" + + context.service_context()->service().apis(0).version(); +} + +std::string GetRulesetTestUri(const context::RequestContext &context, + const std::string &ruleset_id) { + return context.service_context()->config()->GetFirebaseServer() + kV1 + + ruleset_id + kTestQuery; +} + +std::string GetReleaseUrl(const context::RequestContext &context) { + return context.service_context()->config()->GetFirebaseServer() + kV1 + + kProjects + context.service_context()->project_id() + kReleases + + GetReleaseName(context); +} + +std::string GetOperation(const std::string &httpMethod) { + if (httpMethod == kHttpPostMethod) { + return kFirebaseCreateMethod; + } + + if (httpMethod == kHttpGetMethod || httpMethod == kHttpHeadMethod || + httpMethod == kHttpOptionsMethod) { + return kFirebaseGetMethod; + } + + if (httpMethod == kHttpDeleteMethod) { + return kFirebaseDeleteMethod; + } + + return kFirebaseUpdateMethod; +} // An AuthzChecker object is created for every incoming request. It does // authorizaiton by calling Firebase Rules service. class AuthzChecker : public std::enable_shared_from_this { public: - AuthzChecker(std::shared_ptr context, - std::function continuation); + // Constructor + AuthzChecker(ApiManagerEnvInterface *env, + auth::ServiceAccountToken *sa_token); - void Check(); + // Check for Authorization success or failure + void Check(std::shared_ptr context, + std::function continuation); private: - // Helper function to send a http GET request. - void HttpFetch(const std::string &url, const std::string &request_body, + // Helper method that invokes the test firebase service api. + void CallTest(const std::string &ruleset_id, + std::shared_ptr context, + std::function continuation); + + // Parse the respose for GET RELEASE API call + Status ParseReleaseResponse(const std::string &json_str, + std::string *ruleset_id); + + // Parses the response for the TEST API call + Status ParseTestResponse(context::RequestContext &context, + const std::string &json_str); + + // Builds the request body for the TESP API call. + Status BuildTestRequestBody(context::RequestContext &context, + std::string *result_string); + + // Invoke the HTTP call + void HttpFetch(const std::string &url, const std::string &method, + const std::string &request_body, std::function continuation); - // Get Auth token for accessing Firebase Rules service. - const std::string &GetAuthToken(); + // Get the auth token for Firebase service + const std::string &GetAuthToken() { + return sa_token_->GetAuthToken( + auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE); + } - // Request context. - std::shared_ptr context_; + std::shared_ptr GetPtr() { return shared_from_this(); } - // Pointer to access ESP running environment. ApiManagerEnvInterface *env_; - - // The final continuation function. - std::function on_done_; + auth::ServiceAccountToken *sa_token_; }; -AuthzChecker::AuthzChecker(std::shared_ptr context, - std::function continuation) - : context_(context), - env_(context_->service_context()->env()), - on_done_(continuation) {} +AuthzChecker::AuthzChecker(ApiManagerEnvInterface *env, + auth::ServiceAccountToken *sa_token) + : env_(env), sa_token_(sa_token) {} -void AuthzChecker::Check() { +void AuthzChecker::Check( + std::shared_ptr context, + std::function final_continuation) { // TODO: Check service config to see if "useSecurityRules" is specified. // If so, call Firebase Rules service TestRuleset API. + + if (!context->service_context()->IsRulesCheckEnabled() || + context->method() == nullptr || !context->method()->auth()) { + env_->LogDebug("Skipping Firebase Rules checks since it is disabled."); + final_continuation(Status::OK); + return; + } + + // Fetch the Release attributes. + auto checker = GetPtr(); + HttpFetch(GetReleaseUrl(*context), kHttpGetMethod, "", + [context, final_continuation, checker](Status status, + std::string &&body) { + std::string ruleset_id; + if (status.ok()) { + checker->env_->LogDebug( + std::string("GetReleasName succeeded with ") + body); + status = checker->ParseReleaseResponse(body, &ruleset_id); + } else { + checker->env_->LogError(std::string("GetReleaseName for ") + + GetReleaseUrl(*context.get()) + + " with status " + status.ToString()); + status = Status(Code::INTERNAL, kFailedFirebaseReleaseFetch); + } + + // If the parsing of the release body is successful, then call the + // Test Api for firebase rules service. + if (status.ok()) { + checker->CallTest(ruleset_id, context, final_continuation); + } else { + final_continuation(status); + } + }); } -const std::string &AuthzChecker::GetAuthToken() { - // TODO: Get Auth token for accessing Firebase Rules service. - static std::string empty; - return empty; +void AuthzChecker::CallTest(const std::string &ruleset_id, + std::shared_ptr context, + std::function continuation) { + std::string body; + Status status = BuildTestRequestBody(*context.get(), &body); + if (!status.ok()) { + continuation(status); + return; + } + + auto checker = GetPtr(); + HttpFetch(GetRulesetTestUri(*context, ruleset_id), kHttpPostMethod, body, + [context, continuation, checker, ruleset_id](Status status, + std::string &&body) { + + if (status.ok()) { + checker->env_->LogDebug( + std::string("Test API succeeded with ") + body); + status = checker->ParseTestResponse(*context.get(), body); + } else { + checker->env_->LogError(std::string("Test API failed with ") + + status.ToString()); + status = Status(Code::INTERNAL, kFailedFirebaseTest); + } + + continuation(status); + }); +} + +Status AuthzChecker::ParseReleaseResponse(const std::string &json_str, + std::string *ruleset_id) { + grpc_json *json = grpc_json_parse_string_with_len( + const_cast(json_str.data()), json_str.length()); + + if (!json) { + return Status(Code::INVALID_ARGUMENT, kInvalidResponse); + } + + Status status = Status::OK; + const char *id = GetStringValue(json, kRulesetName); + *ruleset_id = (id == nullptr) ? "" : id; + + if (ruleset_id->empty()) { + env_->LogError("Empty ruleset Id received from firebase service"); + status = Status(Code::INTERNAL, kInvalidResponse); + } else { + env_->LogDebug(std::string("Received ruleset Id: ") + *ruleset_id); + } + + grpc_json_destroy(json); + return status; +} + +Status AuthzChecker::ParseTestResponse(context::RequestContext &context, + const std::string &json_str) { + grpc_json *json = grpc_json_parse_string_with_len( + const_cast(json_str.data()), json_str.length()); + + if (!json) { + return Status(Code::INVALID_ARGUMENT, + "Invalid JSON response from Firebase Service"); + } + + Status status = Status::OK; + Status invalid = Status(Code::INTERNAL, kInvalidResponse); + + const grpc_json *testResults = GetProperty(json, kTestResults); + if (testResults == nullptr) { + env_->LogError("TestResults are null"); + status = invalid; + } else { + const char *result = GetStringValue(testResults->child, kState); + if (result == nullptr) { + env_->LogInfo("Result state is empty"); + status = invalid; + } else if (std::string(result) != kTestSuccess) { + status = Status(Code::PERMISSION_DENIED, + std::string("Unauthorized ") + + context.request()->GetRequestHTTPMethod() + + " access to resource " + + context.request()->GetRequestPath(), + Status::AUTH); + } + } + + grpc_json_destroy(json); + return status; +} + +Status AuthzChecker::BuildTestRequestBody(context::RequestContext &context, + std::string *result_string) { + proto::TestRulesetRequest request; + auto *test_case = request.add_test_cases(); + auto httpMethod = context.request()->GetRequestHTTPMethod(); + + test_case->set_service_name(context.service_context()->service_name()); + test_case->set_resource_path(context.request()->GetRequestPath()); + test_case->set_operation(GetOperation(httpMethod)); + test_case->set_expectation(proto::TestRulesetRequest::TestCase::ALLOW); + + ::google::protobuf::Value auth; + ::google::protobuf::Value token; + ::google::protobuf::Value claims; + + Status status = utils::JsonToProto(context.auth_claims(), &claims); + if (!status.ok()) { + env_->LogError(std::string("Error creating Protobuf from claims") + + status.ToString()); + return status; + } + + SetProtoValue(kToken, claims, &token); + SetProtoValue(kAuth, token, &auth); + + auto *variables = test_case->mutable_variables(); + (*variables)[kRequest] = auth; + + status = + utils::ProtoToJson(request, result_string, utils::JsonOptions::DEFAULT); + if (status.ok()) { + env_->LogDebug(std::string("PRotobuf to JSON string = ") + *result_string); + } else { + env_->LogError(std::string("Error creating TestRulesetRequest") + + status.ToString()); + } + + return status; } void AuthzChecker::HttpFetch( - const std::string &url, const std::string &request_body, + const std::string &url, const std::string &method, + const std::string &request_body, std::function continuation) { + env_->LogDebug(std::string("Issue HTTP Request to url :") + url + + " method : " + method + " body: " + request_body); + std::unique_ptr request(new HTTPRequest([continuation]( Status status, std::map &&, std::string &&body) { continuation(status, std::move(body)); })); + if (!request) { continuation(Status(Code::INTERNAL, "Out of memory"), ""); return; } - request->set_method("POST") - .set_url(url) - .set_auth_token(GetAuthToken()) - .set_header("Content-Type", "application/json") - .set_body(request_body); + request->set_method(method).set_url(url).set_auth_token(GetAuthToken()); + + if (method != kHttpGetMethod) { + request->set_header(kContentType, kApplication).set_body(request_body); + } + env_->RunHTTPRequest(std::move(request)); } @@ -97,9 +347,10 @@ void AuthzChecker::HttpFetch( void CheckSecurityRules(std::shared_ptr context, std::function continuation) { - std::shared_ptr authzChecker = - std::make_shared(context, continuation); - authzChecker->Check(); + std::shared_ptr checker = std::make_shared( + context->service_context()->env(), + context->service_context()->service_account_token()); + checker->Check(context, continuation); } } // namespace api_manager diff --git a/contrib/endpoints/src/api_manager/config.cc b/contrib/endpoints/src/api_manager/config.cc index b0a1b85d119..d537ef73e07 100644 --- a/contrib/endpoints/src/api_manager/config.cc +++ b/contrib/endpoints/src/api_manager/config.cc @@ -513,5 +513,13 @@ void Config::SetJwksUri(const string &issuer, const string &jwks_uri, } } +std::string Config::GetFirebaseServer() { + if (server_config_ == nullptr) { + return ""; + } + + return server_config_->api_check_security_rules_config().firebase_server(); +} + } // namespace api_manager } // namespace google diff --git a/contrib/endpoints/src/api_manager/config.h b/contrib/endpoints/src/api_manager/config.h index 9a56d16d745..82586939eb0 100644 --- a/contrib/endpoints/src/api_manager/config.h +++ b/contrib/endpoints/src/api_manager/config.h @@ -78,6 +78,9 @@ class Config { void SetJwksUri(const std::string &issuer, const std::string &jwks_uri, bool openid_valid); + // Get the Firebase server from Server config + std::string GetFirebaseServer(); + private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Config); diff --git a/contrib/endpoints/src/api_manager/config_test.cc b/contrib/endpoints/src/api_manager/config_test.cc index ace0d2afc49..d9170ddf131 100644 --- a/contrib/endpoints/src/api_manager/config_test.cc +++ b/contrib/endpoints/src/api_manager/config_test.cc @@ -870,6 +870,45 @@ TEST(Config, TestCorsDisabled) { ASSERT_EQ(nullptr, method1); } +TEST(Config, TestFirebaseServerCheck) { + MockApiManagerEnvironmentWithLog env; + + static const char server_config[] = R"( +api_check_security_rules_config { + firebase_server: "https://myfirebaseserver.com/" +} +)"; + + std::unique_ptr config = + Config::Create(&env, kServiceNameConfig, server_config); + ASSERT_TRUE(config); + + ASSERT_EQ(config->GetFirebaseServer(), "https://myfirebaseserver.com/"); +} + +TEST(Config, TestEmptyFirebaseServerCheck) { + MockApiManagerEnvironmentWithLog env; + + static const char server_config[] = R"( +service_control_config { + check_aggregator_config { + cache_entries: 1000 + flush_interval_ms: 10 + response_expiration_ms: 20 + } + report_aggregator_config { + cache_entries: 1020 + flush_interval_ms: 15 + } +} +)"; + + std::unique_ptr config = + Config::Create(&env, kServiceNameConfig, server_config); + ASSERT_TRUE(config); + + ASSERT_TRUE(config->GetFirebaseServer().empty()); +} } // namespace } // namespace api_manager diff --git a/contrib/endpoints/src/api_manager/context/request_context.h b/contrib/endpoints/src/api_manager/context/request_context.h index 5b29c271aad..c633952a811 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.h +++ b/contrib/endpoints/src/api_manager/context/request_context.h @@ -37,7 +37,9 @@ class RequestContext { std::unique_ptr request); // Get the ApiManagerImpl object. - context::ServiceContext *service_context() { return service_context_.get(); } + context::ServiceContext *service_context() const { + return service_context_.get(); + } // Get the request object. Request *request() { return request_.get(); } @@ -112,6 +114,10 @@ class RequestContext { last_report_time_ = tp; } + void set_auth_claims(const std::string &claims) { auth_claims_ = claims; } + + const std::string &auth_claims() { return auth_claims_; } + private: // Fill OperationInfo void FillOperationInfo(service_control::OperationInfo *info); @@ -163,6 +169,9 @@ class RequestContext { // Report(). std::string auth_authorized_party_; + // Auth Claims: This is the decoded payload of the JWT token + std::string auth_claims_; + // Used by cloud tracing. std::unique_ptr cloud_trace_; diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index d8fc9dfc409..f6dc3d90275 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -44,6 +44,10 @@ const int kIntermediateReportInterval = 10; const char kHTTPHeadMethod[] = "HEAD"; const char kHTTPGetMethod[] = "GET"; + +const char kFirebaseAudience[] = + "https://staging-firebaserules.sandbox.googleapis.com/" + "google.firebase.rules.v1.FirebaseRulesService"; } ServiceContext::ServiceContext(std::unique_ptr env, @@ -53,10 +57,12 @@ ServiceContext::ServiceContext(std::unique_ptr env, service_account_token_(env_.get()), service_control_(CreateInterface()), cloud_trace_aggregator_(CreateCloudTraceAggregator()), - is_auth_force_disabled_(config_->server_config() && - config_->server_config() - ->api_authentication_config() - .force_disable()) { + is_auth_force_disabled_( + config_->server_config() && + config_->server_config()->has_api_authentication_config() && + config_->server_config() + ->api_authentication_config() + .force_disable()) { intermediate_report_interval_ = kIntermediateReportInterval; // Check server_config override. @@ -69,6 +75,9 @@ ServiceContext::ServiceContext(std::unique_ptr env, ->service_control_config() .intermediate_report_min_interval(); } + + service_account_token_.SetAudience( + auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE, kFirebaseAudience); } MethodCallInfo ServiceContext::GetMethodCallInfo( diff --git a/contrib/endpoints/src/api_manager/context/service_context.h b/contrib/endpoints/src/api_manager/context/service_context.h index 5633ca3dd88..61524813157 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.h +++ b/contrib/endpoints/src/api_manager/context/service_context.h @@ -65,6 +65,11 @@ class ServiceContext { return !is_auth_force_disabled_ && config_->HasAuth(); } + bool IsRulesCheckEnabled() const { + return RequireAuth() && service().apis_size() > 0 && + !config_->GetFirebaseServer().empty(); + } + auth::Certs &certs() { return certs_; } auth::JwtCache &jwt_cache() { return jwt_cache_; } diff --git a/contrib/endpoints/src/api_manager/proto/security_rules.proto b/contrib/endpoints/src/api_manager/proto/security_rules.proto new file mode 100644 index 00000000000..ce8d2690fe0 --- /dev/null +++ b/contrib/endpoints/src/api_manager/proto/security_rules.proto @@ -0,0 +1,57 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +syntax = "proto3"; + +package google.api_manager.proto; + +import "google/protobuf/struct.proto"; + +message TestRulesetRequest { + message TestCase { + // The set of supported test case expectations. + enum Expectation { + EXPECTATION_UNSPECIFIED = 0; // Unspecified expectation. + ALLOW = 1; // Expect an allowed result. + DENY = 2; // Expect a denied result. + } + + // The name of the service that is the subject of the test case. + string service_name = 1; + + // The RESTful resource path of the mock `request`. + string resource_path = 2; + + // The `request` `operation`. The operation will typically be one of `get`, + // `list`, `create`, `update`, or `delete`. Services also may provide custom + // operations. + string operation = 3; + + // Test expectation. + Expectation expectation = 4; + + // (-- + // Variables and fake resources need to be updated to support multiple + // services and the standardized `request` definition. + // --) + + // Optional set of variable values to use during evaluation. + map variables = 5; + } + + // The set of test cases to run against the `Source` if it is well-formed. + repeated TestCase test_cases = 3; +} diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index 343414aceae..9e325c3f1a0 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -39,6 +39,9 @@ message ServerConfig { // Envoy/esp talks to Mixer, has to specify this field. MixerOptions mixer_options = 6; + // Server config used for API authorization via Firebase Rules. + ApiCheckSecurityRulesConfig api_check_security_rules_config = 7; + // Experimental flags Experimental experimental = 999; } @@ -143,6 +146,12 @@ message ApiAuthenticationConfig { bool force_disable = 1; } +// Server config for API Authorization via Firebase Rules +message ApiCheckSecurityRulesConfig { + // Firebase server to use. + string firebase_server = 1; +} + message MixerOptions { // For envoy, it is the cluster name for mixer server. string mixer_server = 1; From b27d4b9fe2e5c17f0c4cdf92c73ee92a76ac02f8 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Fri, 3 Feb 2017 10:35:08 -0800 Subject: [PATCH 04/15] Merge from master into firebase (#65) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect (#38) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect * Fixed style. * Rebase Envoy (#41) * Update prototype to use iptables (#42) * Rebase to fixed Envoy (#43) * Handle HEAD request. (#34) * Handle HEAD request. * Try with GET if HEAD fails. * Address comments. * Format file. * Expose bazel target (#48) * Try again (#49) * Integrate with mixer client. (#55) * Integrate with mixer client. * Restore repositories.bzl back. * Add originIp and originHost attributes. (#56) * Add uuid-dev dependency in README.md (#45) * Extract originIp and OriginHost. (#57) * Extract originIp and OriginHost. * Make header x-forwarded-host const. * Update buckets for UI. (#58) * Update buckets for UI. * Only update time_distribution. * Add targetService attribute. (#59) * Use envoy new access_log handler for sending Report. (#60) * use access_log handler. * Not to use Loggable base class. * Update to the latest envoy with #396. (#61) * Fix tclap dependency fetching error (#62) --- WORKSPACE | 22 +- .../endpoints/include/api_manager/request.h | 2 + contrib/endpoints/repositories.bzl | 52 +-- .../api_manager/context/request_context.cc | 3 +- .../api_manager/context/service_context.cc | 1 - contrib/endpoints/src/api_manager/mixer/BUILD | 6 +- .../endpoints/src/api_manager/mixer/mixer.cc | 315 ++++++++---------- .../endpoints/src/api_manager/mixer/mixer.h | 18 +- .../src/api_manager/service_control/info.h | 8 +- .../src/api_manager/service_control/proto.cc | 2 +- .../testdata/final_report_request.golden | 150 ++++++++- .../testdata/report_request.golden | 150 ++++++++- .../testdata/report_request_failed.golden | 75 ++++- src/envoy/prototype/README.md | 5 + src/envoy/prototype/api_manager_env.cc | 2 +- src/envoy/prototype/api_manager_filter.cc | 124 +++++-- src/envoy/prototype/envoy-esp.conf | 15 +- src/envoy/prototype/server_config.pb.txt | 4 +- src/envoy/repositories.bzl | 4 +- 19 files changed, 650 insertions(+), 308 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index bbafd205918..95dcdde5cdc 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -31,16 +31,34 @@ googletest_repositories() load( "//contrib/endpoints:repositories.bzl", "grpc_repositories", - "mixerapi_repositories", + "mixer_client_repositories", "servicecontrol_client_repositories", ) grpc_repositories() -mixerapi_repositories() +mixer_client_repositories() servicecontrol_client_repositories() +# Workaround for Bazel > 0.4.0 since it needs newer protobuf.bzl from: +# https://github.com/google/protobuf/pull/2246 +# Do not use this git_repository for anything else than protobuf.bzl +new_git_repository( + name = "protobuf_bzl", + # Injecting an empty BUILD file to prevent using any build target + build_file_content = "", + commit = "05090726144b6e632c50f47720ff51049bfcbef6", + remote = "https://github.com/google/protobuf.git", +) + +load( + "@mixerclient_git//:repositories.bzl", + "mixerapi_repositories", +) + +mixerapi_repositories(protobuf_repo="@protobuf_bzl//") + load( "//src/envoy:repositories.bzl", "envoy_repositories", diff --git a/contrib/endpoints/include/api_manager/request.h b/contrib/endpoints/include/api_manager/request.h index ad604b5a41b..8d96ff356d6 100644 --- a/contrib/endpoints/include/api_manager/request.h +++ b/contrib/endpoints/include/api_manager/request.h @@ -43,6 +43,8 @@ class Request { // Gets Client IP // This will be used by service control Check() call. virtual std::string GetClientIP() = 0; + // Gets Client Host. + virtual std::string GetClientHost() { return ""; } // Get GRPC stats. virtual int64_t GetGrpcRequestBytes() = 0; diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index f3df61722a2..b1f14aae8b5 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -334,51 +334,15 @@ def servicecontrol_client_repositories(bind=True): actual = "@servicecontrol_client_git//:service_control_client_lib", ) -def mixerapi_repositories(protobuf_repo="@protobuf_git//", bind=True): - BUILD = """ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -# -licenses(["notice"]) - -load("{}:protobuf.bzl", "cc_proto_library") - -cc_proto_library( - name = "mixer_api_cc_proto", - srcs = glob( - ["mixer/api/v1/*.proto"], - ), - default_runtime = "//external:protobuf", - protoc = "//external:protoc", - visibility = ["//visibility:public"], - deps = [ - "//external:cc_wkt_protos", - "//external:servicecontrol", - ], -) -""".format(protobuf_repo) - - native.new_git_repository( - name = "mixerapi_git", - commit = "fc5a396185edc72d06d1937f30a8148a37d4fc1b", - remote = "https://github.com/istio/api.git", - build_file_content = BUILD, +def mixer_client_repositories(bind=True): + native.git_repository( + name = "mixerclient_git", + commit = "80e450a5126960e8e6337c3631cf2ef984038eab", + remote = "https://github.com/istio/mixerclient.git", ) + if bind: native.bind( - name = "mixer_api_cc_proto", - actual = "@mixerapi_git//:mixer_api_cc_proto", + name = "mixer_client_lib", + actual = "@mixerclient_git//:mixer_client_lib", ) diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index cbf4e925e37..59cc7c3d53f 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -171,6 +171,8 @@ void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { info->producer_project_id = service_context()->project_id(); info->referer = http_referer_; info->request_start_time = start_time_; + info->client_ip = request_->GetClientIP(); + info->client_host = request_->GetClientHost(); } void RequestContext::FillLocation(service_control::ReportRequestInfo *info) { @@ -221,7 +223,6 @@ void RequestContext::FillLogMessage(service_control::ReportRequestInfo *info) { void RequestContext::FillCheckRequestInfo( service_control::CheckRequestInfo *info) { FillOperationInfo(info); - info->client_ip = request_->GetClientIP(); info->allow_unregistered_calls = method()->allow_unregistered_calls(); } diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index f6dc3d90275..0e99c247676 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -44,7 +44,6 @@ const int kIntermediateReportInterval = 10; const char kHTTPHeadMethod[] = "HEAD"; const char kHTTPGetMethod[] = "GET"; - const char kFirebaseAudience[] = "https://staging-firebaserules.sandbox.googleapis.com/" "google.firebase.rules.v1.FirebaseRulesService"; diff --git a/contrib/endpoints/src/api_manager/mixer/BUILD b/contrib/endpoints/src/api_manager/mixer/BUILD index 8082c61e201..affe09c7ddd 100644 --- a/contrib/endpoints/src/api_manager/mixer/BUILD +++ b/contrib/endpoints/src/api_manager/mixer/BUILD @@ -32,11 +32,11 @@ cc_library( ], }), deps = [ - "//external:grpc++", - "//external:mixer_api_cc_proto", - "//external:protobuf", "//contrib/endpoints/src/api_manager:impl_headers", "//contrib/endpoints/src/api_manager/service_control", "//contrib/endpoints/src/api_manager/utils", + "//external:grpc++", + "//external:mixer_client_lib", + "//external:protobuf", ], ) diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc index 0c0bb5d793e..53837edaf67 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ b/contrib/endpoints/src/api_manager/mixer/mixer.cc @@ -14,169 +14,154 @@ */ #include "contrib/endpoints/src/api_manager/mixer/mixer.h" -#include -#include "mixer/api/v1/service.pb.h" - using ::google::api_manager::utils::Status; -using ::google::protobuf::util::error::Code; -using ::google::protobuf::Map; +using ::istio::mixer_client::Attributes; namespace google { namespace api_manager { namespace mixer { namespace { -const char kMixerServiceName[] = "istio.mixer.v1.Mixer"; - -enum AttributeIndex { - ATTR_SERVICE_NAME = 0, - ATTR_PEER_ID, - ATTR_OPERATION_NAME, - ATTR_API_KEY, - ATTR_RESPONSE_CODE, - ATTR_URL, - ATTR_LOCATION, - ATTR_API_NAME, - ATTR_API_VERSION, - ATTR_API_METHOD, - ATTR_REQUEST_SIZE, - ATTR_RESPONSE_SIZE, - ATTR_LOG_MESSAGE, -}; - -struct AttributeDict { - int index; - std::string name; -} kAttributeNames[] = { - { - ATTR_SERVICE_NAME, "serviceName", - }, - { - ATTR_PEER_ID, "peerId", - }, - { - ATTR_OPERATION_NAME, "operationName", - }, - { - ATTR_API_KEY, "apiKey", - }, - { - ATTR_RESPONSE_CODE, "responseCode", - }, - { - ATTR_URL, "URL", - }, - { - ATTR_LOCATION, "location", - }, - { - ATTR_API_NAME, "apiName", - }, - { - ATTR_API_VERSION, "apiVersion", - }, - { - ATTR_API_METHOD, "apiMethod", - }, - { - ATTR_REQUEST_SIZE, "requestSize", - }, - { - ATTR_RESPONSE_SIZE, "responesSize", - }, - { - ATTR_LOG_MESSAGE, "logMessage", - }, -}; - -void SetAttributeDict(Map* dict) { - for (auto attr : kAttributeNames) { - (*dict)[attr.index] = attr.name; - } +const std::string kProxyPeerID = "Istio/Proxy"; +const std::string kEnvNameTargetService = "TARGET_SERVICE"; + +const std::string kAttrNameServiceName = "serviceName"; +const std::string kAttrNamePeerId = "peerId"; +const std::string kAttrNameOperationId = "operationId"; +const std::string kAttrNameOperationName = "operationName"; +const std::string kAttrNameApiKey = "apiKey"; +const std::string kAttrNameResponseCode = "responseCode"; +const std::string kAttrNameURL = "url"; +const std::string kAttrNameLocation = "location"; +const std::string kAttrNameApiName = "apiName"; +const std::string kAttrNameApiVersion = "apiVersion"; +const std::string kAttrNameApiMethod = "apiMethod"; +const std::string kAttrNameRequestSize = "requestSize"; +const std::string kAttrNameResponseSize = "responseSize"; +const std::string kAttrNameLogMessage = "logMessage"; +const std::string kAttrNameResponseTime = "responseTime"; +const std::string kAttrNameOriginIp = "originIp"; +const std::string kAttrNameOriginHost = "originHost"; +const std::string kAttrNameTargetService = "targetService"; + +Attributes::Value StringValue(const std::string& str) { + Attributes::Value v; + v.type = Attributes::Value::STRING; + v.str_v = str; + return v; } -void CovertToPb(const service_control::CheckRequestInfo& info, - const std::string& service_name, - ::istio::mixer::v1::Attributes* attr) { - SetAttributeDict(attr->mutable_dictionary()); - - auto* str_attrs = attr->mutable_string_attributes(); - (*str_attrs)[ATTR_SERVICE_NAME] = service_name; - (*str_attrs)[ATTR_PEER_ID] = "Proxy"; - (*str_attrs)[ATTR_OPERATION_NAME] = info.operation_name; - (*str_attrs)[ATTR_API_KEY] = info.api_key; +Attributes::Value Int64Value(int64_t value) { + Attributes::Value v; + v.type = Attributes::Value::INT64; + v.value.int64_v = value; + return v; } -void CovertToPb(const service_control::ReportRequestInfo& info, - const std::string& service_name, - ::istio::mixer::v1::Attributes* attr) { - SetAttributeDict(attr->mutable_dictionary()); +} // namespace - auto* str_attrs = attr->mutable_string_attributes(); - (*str_attrs)[ATTR_SERVICE_NAME] = service_name; - (*str_attrs)[ATTR_PEER_ID] = "Proxy"; - (*str_attrs)[ATTR_OPERATION_NAME] = info.operation_name; - (*str_attrs)[ATTR_API_KEY] = info.api_key; +Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) + : env_(env), config_(config) {} - (*str_attrs)[ATTR_URL] = info.url; - (*str_attrs)[ATTR_LOCATION] = info.location; +Mixer::~Mixer() {} - (*str_attrs)[ATTR_API_NAME] = info.api_name; - (*str_attrs)[ATTR_API_VERSION] = info.api_version; - (*str_attrs)[ATTR_API_METHOD] = info.api_method; +Status Mixer::Init() { + ::istio::mixer_client::MixerClientOptions options; + options.mixer_server = + config_->server_config()->mixer_options().mixer_server(); + mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + auto target_service = getenv(kEnvNameTargetService.c_str()); + if (target_service) { + target_service_ = target_service; + } + return Status::OK; +} - (*str_attrs)[ATTR_LOG_MESSAGE] = info.log_message; +Status Mixer::Close() { return Status::OK; } + +void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, + ::istio::mixer_client::Attributes* attr) { + attr->attributes[kAttrNameServiceName] = StringValue(config_->service_name()); + attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); + + if (!info.operation_id.empty()) { + attr->attributes[kAttrNameOperationId] = StringValue(info.operation_id); + } + if (!info.operation_name.empty()) { + attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); + } + if (!info.api_key.empty()) { + attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); + } + if (!info.client_ip.empty()) { + attr->attributes[kAttrNameOriginIp] = StringValue(info.client_ip); + } + if (!info.client_host.empty()) { + attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); + } + if (!target_service_.empty()) { + attr->attributes[kAttrNameTargetService] = StringValue(target_service_); + } +} - auto* int_attrs = attr->mutable_int64_attributes(); - (*int_attrs)[ATTR_RESPONSE_CODE] = info.response_code; - (*int_attrs)[ATTR_REQUEST_SIZE] = info.request_size; - (*int_attrs)[ATTR_RESPONSE_SIZE] = info.response_size; +void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, + ::istio::mixer_client::Attributes* attr) { + FillCommonAttributes(info, attr); } -} // namespace +void Mixer::FillReportAttributes(const service_control::ReportRequestInfo& info, + ::istio::mixer_client::Attributes* attr) { + FillCommonAttributes(info, attr); -Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) - : env_(env), request_index_(0), config_(config) {} + if (!info.url.empty()) { + attr->attributes[kAttrNameURL] = StringValue(info.url); + } + if (!info.location.empty()) { + attr->attributes[kAttrNameLocation] = StringValue(info.location); + } -Mixer::~Mixer() {} + if (!info.api_name.empty()) { + attr->attributes[kAttrNameApiName] = StringValue(info.api_name); + } + if (!info.api_version.empty()) { + attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); + } + if (!info.api_method.empty()) { + attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); + } -Status Mixer::Init() { return Status::OK; } + if (!info.log_message.empty()) { + attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); + } -Status Mixer::Close() { return Status::OK; } + attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); + if (info.request_size >= 0) { + attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); + } + if (info.response_size >= 0) { + attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); + } + + if (info.latency.request_time_ms >= 0) { + attr->attributes[kAttrNameResponseTime] = + Int64Value(info.latency.request_time_ms); + } +} Status Mixer::Report(const service_control::ReportRequestInfo& info) { - std::unique_ptr grpc_request(new GRPCRequest([this]( - Status status, std::string&& body) { - if (status.ok()) { - // Handle 200 response - ::istio::mixer::v1::ReportResponse response; - if (!response.ParseFromString(body)) { - status = - Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); - env_->LogError(std::string("Failed parse report response: ") + body); - } - env_->LogInfo(std::string("Report response: ") + response.DebugString()); - } else { - env_->LogError(std::string("Failed to call Mixer::report, Error: ") + - status.ToString()); - } - })); - - ::istio::mixer::v1::ReportRequest request; - request.set_request_index(++request_index_); - CovertToPb(info, config_->service_name(), request.mutable_attribute_update()); - env_->LogInfo(std::string("Send Report: ") + request.DebugString()); - - std::string request_body; - request.SerializeToString(&request_body); - - grpc_request - ->set_server(config_->server_config()->mixer_options().mixer_server()) - .set_service(kMixerServiceName) - .set_method("Report") - .set_body(request_body); - - env_->RunGRPCRequest(std::move(grpc_request)); + ::istio::mixer_client::Attributes attributes; + FillReportAttributes(info, &attributes); + env_->LogInfo("Send Report: "); + env_->LogInfo(attributes.DebugString()); + mixer_client_->Report( + attributes, [this](const ::google::protobuf::util::Status& status) { + if (status.ok()) { + env_->LogInfo("Report response: OK"); + } else { + env_->LogError(std::string("Failed to call Mixer::report, Error: ") + + status.ToString()); + } + }); return Status::OK; } @@ -185,40 +170,24 @@ void Mixer::Check( cloud_trace::CloudTraceSpan* parent_span, std::function on_done) { - std::unique_ptr grpc_request(new GRPCRequest([this, on_done]( - Status status, std::string&& body) { - if (status.ok()) { - // Handle 200 response - ::istio::mixer::v1::CheckResponse response; - if (!response.ParseFromString(body)) { - status = - Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); - env_->LogError(std::string("Failed parse check response: ") + body); - } - env_->LogInfo(std::string("Check response: ") + response.DebugString()); - } else { - env_->LogError(std::string("Failed to call Mixer::check, Error: ") + - status.ToString()); - } - service_control::CheckResponseInfo info; - on_done(status, info); - })); - - ::istio::mixer::v1::CheckRequest request; - request.set_request_index(++request_index_); - CovertToPb(info, config_->service_name(), request.mutable_attribute_update()); - env_->LogInfo(std::string("Send Check: ") + request.DebugString()); - - std::string request_body; - request.SerializeToString(&request_body); - - grpc_request - ->set_server(config_->server_config()->mixer_options().mixer_server()) - .set_service(kMixerServiceName) - .set_method("Check") - .set_body(request_body); - - env_->RunGRPCRequest(std::move(grpc_request)); + ::istio::mixer_client::Attributes attributes; + FillCheckAttributes(info, &attributes); + env_->LogInfo("Send Check: "); + env_->LogInfo(attributes.DebugString()); + mixer_client_->Check( + attributes, + [this, on_done](const ::google::protobuf::util::Status& status) { + if (status.ok()) { + env_->LogInfo("Check response: OK"); + } else { + env_->LogError(std::string("Failed to call Mixer::check, Error: ") + + status.ToString()); + } + service_control::CheckResponseInfo info; + on_done(Status(status.error_code(), status.error_message(), + Status::SERVICE_CONTROL), + info); + }); } Status Mixer::GetStatistics(service_control::Statistics* esp_stat) const { diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h index 0318a5f06f1..81276e1227d 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ b/contrib/endpoints/src/api_manager/mixer/mixer.h @@ -18,6 +18,7 @@ #include "contrib/endpoints/include/api_manager/env_interface.h" #include "contrib/endpoints/src/api_manager/config.h" #include "contrib/endpoints/src/api_manager/service_control/interface.h" +#include "include/client.h" namespace google { namespace api_manager { @@ -49,11 +50,24 @@ class Mixer : public service_control::Interface { // The constructor. Mixer(ApiManagerEnvInterface* env, const Config* config); + // Fill common attributes for both check and report. + void FillCommonAttributes(const service_control::OperationInfo& info, + ::istio::mixer_client::Attributes* attr); + // Fill attributes for check. + void FillCheckAttributes(const service_control::CheckRequestInfo& info, + ::istio::mixer_client::Attributes* attr); + // Fill attributes for report. + void FillReportAttributes(const service_control::ReportRequestInfo& info, + ::istio::mixer_client::Attributes* attr); + // The Api Manager environment interface. ApiManagerEnvInterface* env_; - int64_t request_index_; - + // The config. const Config* config_; + // The mixer client + std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // Target service + std::string target_service_; }; } // namespace mixer diff --git a/contrib/endpoints/src/api_manager/service_control/info.h b/contrib/endpoints/src/api_manager/service_control/info.h index 15364ad58c6..f203057cc9e 100644 --- a/contrib/endpoints/src/api_manager/service_control/info.h +++ b/contrib/endpoints/src/api_manager/service_control/info.h @@ -60,13 +60,17 @@ struct OperationInfo { // and Report. std::chrono::system_clock::time_point request_start_time; + // The client IP address. + std::string client_ip; + + // The client host name. + std::string client_host; + OperationInfo() {} }; // Information to fill Check request protobuf. struct CheckRequestInfo : public OperationInfo { - // The client IP address. - std::string client_ip; // Whether the method allow unregistered calls. bool allow_unregistered_calls; diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 17b921db076..3e7eba64d21 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -96,7 +96,7 @@ struct DistributionHelperOptions { double scale; }; -const DistributionHelperOptions time_distribution = {8, 10.0, 1e-6}; +const DistributionHelperOptions time_distribution = {29, 2.0, 1e-6}; const DistributionHelperOptions size_distribution = {8, 10.0, 1}; const double kMsToSecs = 1e-3; diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden index b9eff9f28eb..d60b039987d 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden @@ -201,13 +201,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -227,13 +248,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -253,13 +295,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -279,13 +342,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -304,14 +388,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -330,14 +435,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden index 5d8a266ef88..faf96383418 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden @@ -213,13 +213,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -239,13 +260,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -265,13 +307,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -291,13 +354,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -316,14 +400,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -342,14 +447,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden index a3ffd58c30d..710772cd215 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden @@ -152,13 +152,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -178,13 +199,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -203,14 +245,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md index 00e73a169d6..dc73f63f6c9 100644 --- a/src/envoy/prototype/README.md +++ b/src/envoy/prototype/README.md @@ -1,6 +1,11 @@ This Proxy will use Envoy and talk to Mixer server. +## Install dependencies + +``` + apt-get install uuid-dev +``` ## Build Mixer server diff --git a/src/envoy/prototype/api_manager_env.cc b/src/envoy/prototype/api_manager_env.cc index 6b1619a6faa..f8536bcc10a 100644 --- a/src/envoy/prototype/api_manager_env.cc +++ b/src/envoy/prototype/api_manager_env.cc @@ -103,7 +103,7 @@ class HTTPRequest : public Http::Message { virtual void body(Buffer::InstancePtr &&body) override {} virtual HeaderMap *trailers() override { return nullptr; } virtual void trailers(HeaderMapPtr &&trailers) override {} - virtual std::string bodyAsString() override { return ""; } + virtual std::string bodyAsString() const override { return ""; } }; class HTTPRequestCallbacks : public AsyncClient::Callbacks { diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 2fca48de23c..3983a4d8b41 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -13,6 +13,12 @@ namespace Http { namespace ApiManager { +namespace { + +// Define lower case string for X-Forwarded-Host. +const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); + +} // namespace std::string ReadFile(const std::string& file_name) { std::ifstream t(file_name); @@ -65,12 +71,18 @@ typedef std::shared_ptr ConfigPtr; class Request : public google::api_manager::Request { private: HeaderMap& header_map_; + std::string downstream_address_; + std::string virtual_host_; bool query_parsed_; std::map query_params_; public: - Request(HeaderMap& header_map) - : header_map_(header_map), query_parsed_(false) {} + Request(HeaderMap& header_map, const std::string& downstream_address, + const std::string& virtual_host) + : header_map_(header_map), + downstream_address_(downstream_address), + virtual_host_(virtual_host), + query_parsed_(false) {} virtual std::string GetRequestHTTPMethod() override { return header_map_.Method()->value().c_str(); } @@ -80,7 +92,31 @@ class Request : public google::api_manager::Request { virtual std::string GetUnparsedRequestPath() override { return header_map_.Path()->value().c_str(); } - virtual std::string GetClientIP() override { return ""; } + + virtual std::string GetClientIP() override { + if (!header_map_.ForwardedFor()) { + return downstream_address_; + } + std::vector xff_address_list = + StringUtil::split(header_map_.ForwardedFor()->value().c_str(), ','); + if (xff_address_list.empty()) { + return downstream_address_; + } + return xff_address_list.front(); + } + + virtual std::string GetClientHost() override { + const HeaderEntry* entry = header_map_.get(kHeaderNameXFH); + if (entry == nullptr) { + return virtual_host_; + } + auto xff_list = StringUtil::split(entry->value().c_str(), ','); + if (xff_list.empty()) { + return virtual_host_; + } + return xff_list.back(); + } + virtual bool FindQuery(const std::string& name, std::string* query) override { if (!query_parsed_) { auto header = header_map_.Path(); @@ -127,24 +163,30 @@ class Request : public google::api_manager::Request { }; class Response : public google::api_manager::Response { + const AccessLog::RequestInfo& request_info_; + + public: + Response(const AccessLog::RequestInfo& request_info) + : request_info_(request_info) {} + google::api_manager::utils::Status GetResponseStatus() { return google::api_manager::utils::Status::OK; } - std::size_t GetRequestSize() { return 0; } + std::size_t GetRequestSize() { return request_info_.bytesReceived(); } - std::size_t GetResponseSize() { return 0; } + std::size_t GetResponseSize() { return request_info_.bytesSent(); } google::api_manager::utils::Status GetLatencyInfo( google::api_manager::service_control::LatencyInfo* info) { + info->request_time_ms = request_info_.duration().count(); return google::api_manager::utils::Status::OK; } }; const Http::HeaderMapImpl BadRequest{{Http::Headers::get().Status, "400"}}; -class Instance : public Http::StreamFilter, - public Logger::Loggable { +class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { private: std::shared_ptr api_manager_; std::unique_ptr @@ -158,18 +200,28 @@ class Instance : public Http::StreamFilter, bool initiating_call_; + std::string getRouteVirtualHost(HeaderMap& headers) const { + const Router::Route* route = decoder_callbacks_->route(); + if (route && route->routeEntry()) { + return route->routeEntry()->virtualHost().name(); + } + return ""; + } + public: Instance(ConfigPtr config) : api_manager_(config->api_manager()), state_(NotStarted), initiating_call_(false) { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); } FilterHeadersStatus decodeHeaders(HeaderMap& headers, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr request(new Request(headers)); + Log().debug("Called ApiManager::Instance : {}", __func__); + std::unique_ptr request( + new Request(headers, decoder_callbacks_->downstreamAddress(), + getRouteVirtualHost(headers))); request_handler_ = api_manager_->CreateRequestHandler(std::move(request)); state_ = Calling; initiating_call_ = true; @@ -181,13 +233,13 @@ class Instance : public Http::StreamFilter, if (state_ == Complete) { return FilterHeadersStatus::Continue; } - log().debug("Called ApiManager::Instance : {} Stop", __func__); + Log().debug("Called ApiManager::Instance : {} Stop", __func__); return FilterHeadersStatus::StopIteration; } FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override { - log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, + Log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, data.length(), end_stream); if (state_ == Calling) { return FilterDataStatus::StopIterationAndBuffer; @@ -196,7 +248,7 @@ class Instance : public Http::StreamFilter, } FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); if (state_ == Calling) { return FilterTrailersStatus::StopIteration; } @@ -204,52 +256,62 @@ class Instance : public Http::StreamFilter, } void setDecoderFilterCallbacks( StreamDecoderFilterCallbacks& callbacks) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); decoder_callbacks_ = &callbacks; decoder_callbacks_->addResetStreamCallback( [this]() { state_ = Responded; }); } void completeCheck(const google::api_manager::utils::Status& status) { - log().debug("Called ApiManager::Instance : check complete {}", + Log().debug("Called ApiManager::Instance : check complete {}", status.ToJson()); if (!status.ok() && state_ != Responded) { state_ = Responded; - Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), - status.ToJson()); + decoder_callbacks_->dispatcher().post([this, status]() { + Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), + status.ToJson()); + }); return; } state_ = Complete; if (!initiating_call_) { - decoder_callbacks_->continueDecoding(); + decoder_callbacks_->dispatcher().post( + [this]() { decoder_callbacks_->continueDecoding(); }); } } virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterHeadersStatus::Continue; } virtual FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterDataStatus::Continue; } virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterTrailersStatus::Continue; } virtual void setEncoderFilterCallbacks( StreamEncoderFilterCallbacks& callbacks) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); encoder_callbacks_ = &callbacks; } - // note: cannot extend ~ActiveStream for access log, placing it here - ~Instance() { - log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr response(new Response()); - request_handler_->Report(std::move(response), - [this]() { log().debug("Report returns"); }); + virtual void log(const HeaderMap* request_headers, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info) override { + Log().debug("Called ApiManager::Instance : {}", __func__); + std::unique_ptr response( + new Response(request_info)); + request_handler_->Report(std::move(response), []() {}); + } + + spdlog::logger& Log() { + static spdlog::logger& instance = + Logger::Registry::getLog(Logger::Id::http); + return instance; } }; } @@ -271,8 +333,10 @@ class ApiManagerConfig : public HttpFilterConfigFactory { new Http::ApiManager::Config(config, server)); return [api_manager_config]( Http::FilterChainFactoryCallbacks& callbacks) -> void { - auto instance = new Http::ApiManager::Instance(api_manager_config); - callbacks.addStreamFilter(Http::StreamFilterPtr{instance}); + std::shared_ptr instance( + new Http::ApiManager::Instance(api_manager_config)); + callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); + callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); }; } }; diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf index 11690bed565..27ea9f1b397 100644 --- a/src/envoy/prototype/envoy-esp.conf +++ b/src/envoy/prototype/envoy-esp.conf @@ -73,18 +73,6 @@ } ] }, - { - "name": "mixer_server", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "features": "http2", - "hosts": [ - { - "url": "tcp://localhost:9091" - } - ] - }, { "name": "api_manager", "connect_timeout_ms": 5000, @@ -97,6 +85,5 @@ ] } ] - }, - "tracing_enabled": "true" + } } diff --git a/src/envoy/prototype/server_config.pb.txt b/src/envoy/prototype/server_config.pb.txt index 9703886cc9e..19cb6cbb2c0 100644 --- a/src/envoy/prototype/server_config.pb.txt +++ b/src/envoy/prototype/server_config.pb.txt @@ -2,5 +2,5 @@ cloud_tracing_config { force_disable: true } mixer_options { - mixer_server: "mixer_server" -} \ No newline at end of file + mixer_server: "localhost:9091" +} diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index dd36a23d456..94f371ebc7d 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -221,7 +221,7 @@ cc_library( native.new_http_archive( name = "tclap_tar", - url = "https://sourceforge.net/projects/tclap/files/tclap-1.2.1.tar.gz/download", + url = "https://storage.googleapis.com/istio-build-deps/tclap-1.2.1.tar.gz", type = "tar.gz", strip_prefix = "tclap-1.2.1", build_file_content = BUILD, @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "c36bb76e304ab4c5584a07303e2953a170888319", + commit = "02c6fc97b4c21d25ab596a25208fbe283e927f6a", build_file_content = BUILD, ) From e675ee3001fb402828dcd93fd64744fe565d1966 Mon Sep 17 00:00:00 2001 From: Tao Li Date: Mon, 6 Feb 2017 10:31:38 -0800 Subject: [PATCH 05/15] Update the auth checke to use service.experimental.authorization.providerwq! --- .idea/misc.xml | 14 + .idea/modules.xml | 8 + .idea/proxy.iml | 13 + .idea/vcs.xml | 6 + .idea/workspace.xml | 354 +++++++++++++++++++++ contrib/endpoints/src/api_manager/config.h | 5 +- 6 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/proxy.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000000..3eb495b0f9e --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000000..e8e95353b28 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/proxy.iml b/.idea/proxy.iml new file mode 100644 index 00000000000..6774f34d4ba --- /dev/null +++ b/.idea/proxy.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000000..94a25f7f4cb --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 00000000000..b69565637e5 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,354 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + DEFINITION_ORDER + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1486405670626 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrib/endpoints/src/api_manager/config.h b/contrib/endpoints/src/api_manager/config.h index 82586939eb0..a825ab5be62 100644 --- a/contrib/endpoints/src/api_manager/config.h +++ b/contrib/endpoints/src/api_manager/config.h @@ -64,8 +64,9 @@ class Config { // TODO: Remove in favor of service(). const std::string &service_name() const { return service_.name(); } - // TODO: Remove in favor of service(). - bool HasAuth() const { return service_.has_authentication(); } + bool HasAuth() const { return service_.has_experimental() && + service_.experimental().has_authorization() && + service_.experimental().authorization().has_provider(); } // Returns true if the caller should try openId discovery to fetch jwksUri. // url is set to the openId discovery link in this case. Returns false From dee3881021a3afdbc76c8d963e27a0cc23088245 Mon Sep 17 00:00:00 2001 From: Tao Li Date: Mon, 6 Feb 2017 11:55:46 -0800 Subject: [PATCH 06/15] Update the auth check to use service.experimental.authorization.provider --- contrib/endpoints/src/api_manager/context/service_context.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/endpoints/src/api_manager/context/service_context.h b/contrib/endpoints/src/api_manager/context/service_context.h index 61524813157..ea215567136 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.h +++ b/contrib/endpoints/src/api_manager/context/service_context.h @@ -67,7 +67,10 @@ class ServiceContext { bool IsRulesCheckEnabled() const { return RequireAuth() && service().apis_size() > 0 && - !config_->GetFirebaseServer().empty(); + (!config_->GetFirebaseServer().empty() || + (service().has_experimental() && + service().experimental().has_authorization() && + service().experimental().authorization().has_provider())); } auth::Certs &certs() { return certs_; } From 0c001dfe8eeb87e1d628e52121d6b1f42f5cfe29 Mon Sep 17 00:00:00 2001 From: wattli Date: Wed, 8 Feb 2017 09:38:20 -0800 Subject: [PATCH 07/15] Update the auth check to use service.experimental.authorization.provider (#67) * Update the auth check to use service.experimental.authorization.provider * Address comments and revert accidental change. * Remove unnecessary added accidentally. * Another patch * fix the logic * fix lint * Fix broken test and add unit tests * Fix comments * Fix style check * revert style for raw string * fix small lint * fix small lint * fix small lint --- .idea/misc.xml | 14 - .idea/modules.xml | 8 - .idea/proxy.iml | 13 - .idea/vcs.xml | 6 - .idea/workspace.xml | 354 ------------------ contrib/endpoints/repositories.bzl | 4 +- contrib/endpoints/src/api_manager/config.cc | 16 +- contrib/endpoints/src/api_manager/config.h | 4 +- .../endpoints/src/api_manager/config_test.cc | 132 ++++--- .../src/api_manager/context/service_context.h | 5 +- 10 files changed, 100 insertions(+), 456 deletions(-) delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/proxy.iml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 3eb495b0f9e..00000000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index e8e95353b28..00000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/proxy.iml b/.idea/proxy.iml deleted file mode 100644 index 6774f34d4ba..00000000000 --- a/.idea/proxy.iml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f4cb..00000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index b69565637e5..00000000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,354 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - DEFINITION_ORDER - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1486405670626 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index b1f14aae8b5..bed9315a267 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -254,6 +254,8 @@ cc_proto_library( "google/api/control.proto", "google/api/documentation.proto", "google/api/endpoint.proto", + "google/api/experimental/authorization_config.proto", + "google/api/experimental/experimental.proto", "google/api/http.proto", "google/api/label.proto", "google/api/log.proto", @@ -293,7 +295,7 @@ cc_proto_library( native.new_git_repository( name = "googleapis_git", - commit = "db1d4547dc56a798915e0eb2c795585385922165", + commit = "412867fb105722fb9d2cd9af90af1f8f120de238", remote = "https://github.com/googleapis/googleapis.git", build_file_content = BUILD, ) diff --git a/contrib/endpoints/src/api_manager/config.cc b/contrib/endpoints/src/api_manager/config.cc index d537ef73e07..66ac9aff1c1 100644 --- a/contrib/endpoints/src/api_manager/config.cc +++ b/contrib/endpoints/src/api_manager/config.cc @@ -514,11 +514,21 @@ void Config::SetJwksUri(const string &issuer, const string &jwks_uri, } std::string Config::GetFirebaseServer() { - if (server_config_ == nullptr) { - return ""; + // Server config overwrites service config. + if (server_config_ != nullptr && + server_config_->has_api_check_security_rules_config() && + !server_config_->api_check_security_rules_config() + .firebase_server() + .empty()) { + return server_config_->api_check_security_rules_config().firebase_server(); } - return server_config_->api_check_security_rules_config().firebase_server(); + if (service_.has_experimental() && + service_.experimental().has_authorization() && + !service_.experimental().authorization().provider().empty()) { + return service_.experimental().authorization().provider(); + } + return ""; } } // namespace api_manager diff --git a/contrib/endpoints/src/api_manager/config.h b/contrib/endpoints/src/api_manager/config.h index a825ab5be62..f7cca3838ff 100644 --- a/contrib/endpoints/src/api_manager/config.h +++ b/contrib/endpoints/src/api_manager/config.h @@ -64,9 +64,7 @@ class Config { // TODO: Remove in favor of service(). const std::string &service_name() const { return service_.name(); } - bool HasAuth() const { return service_.has_experimental() && - service_.experimental().has_authorization() && - service_.experimental().authorization().has_provider(); } + bool HasAuth() const { return service_.has_authentication(); } // Returns true if the caller should try openId discovery to fetch jwksUri. // url is set to the openId discovery link in this case. Returns false diff --git a/contrib/endpoints/src/api_manager/config_test.cc b/contrib/endpoints/src/api_manager/config_test.cc index d9170ddf131..2302bd5cbf7 100644 --- a/contrib/endpoints/src/api_manager/config_test.cc +++ b/contrib/endpoints/src/api_manager/config_test.cc @@ -501,35 +501,34 @@ TEST(Config, LoadBackends) { TEST(Config, RpcMethodsWithHttpRules) { MockApiManagerEnvironmentWithLog env; - const char config_text[] = - R"( - name : "BookstoreApi" - apis { - name: "Bookstore" - methods { - name: "ListShelves" - request_type_url: "types.googleapis.com/google.protobuf.Empty" - response_type_url: "types.googleapis.com/Bookstore.ListShelvesResponse" - } - methods { - name: "CreateShelves" - request_streaming: true - request_type_url: "types.googleapis.com/Bookstore.Shelf" - response_streaming: true - response_type_url: "types.googleapis.com/Bookstore.Shelf" - } + const char config_text[] = R"( + name : "BookstoreApi" + apis { + name: "Bookstore" + methods { + name: "ListShelves" + request_type_url: "types.googleapis.com/google.protobuf.Empty" + response_type_url: "types.googleapis.com/Bookstore.ListShelvesResponse" } - http { - rules { - selector: "Bookstore.ListShelves" - get: "/shelves" - } - rules { - selector: "Bookstore.CreateShelves" - post: "/shelves" - } + methods { + name: "CreateShelves" + request_streaming: true + request_type_url: "types.googleapis.com/Bookstore.Shelf" + response_streaming: true + response_type_url: "types.googleapis.com/Bookstore.Shelf" } - )"; + } + http { + rules { + selector: "Bookstore.ListShelves" + get: "/shelves" + } + rules { + selector: "Bookstore.CreateShelves" + post: "/shelves" + } + } + )"; std::unique_ptr config = Config::Create(&env, config_text, ""); ASSERT_TRUE(config); @@ -764,8 +763,8 @@ TEST(Config, TestHttpOptions) { rules { selector: "CorsShelves" custom: { - kind: "OPTIONS" - path: "/shelves" + kind: "OPTIONS" + path: "/shelves" } } rules { @@ -870,44 +869,77 @@ TEST(Config, TestCorsDisabled) { ASSERT_EQ(nullptr, method1); } -TEST(Config, TestFirebaseServerCheck) { +static const char kServiceConfigWithoutAuthz[] = R"( + name: "Service.Name" +)"; + +static const char kServiceConfigWithAuthz[] = R"( + name: "Service.Name" + experimental { + authorization { + provider: "authz@firebase.com" + } + } +)"; + +static const char kServerConfigWithoutAuthz[] = R"( + service_control_config { + check_aggregator_config { + cache_entries: 1000 + flush_interval_ms: 10 + response_expiration_ms: 20 + } + report_aggregator_config { + cache_entries: 1020 + flush_interval_ms: 15 + } + } +)"; + +static const char kServerConfigWithAuthz[] = R"( + api_check_security_rules_config { + firebase_server: "https://myfirebaseserver.com/" + } +)"; + +TEST(Config, TestFirebaseServerCheckWithServiceAuthzWithoutServerAuthz) { MockApiManagerEnvironmentWithLog env; - static const char server_config[] = R"( -api_check_security_rules_config { - firebase_server: "https://myfirebaseserver.com/" + std::unique_ptr config = + Config::Create(&env, kServiceConfigWithAuthz, kServerConfigWithoutAuthz); + ASSERT_TRUE(config); + + ASSERT_EQ(config->GetFirebaseServer(), "authz@firebase.com"); } -)"; + +TEST(Config, TestFirebaseServerCheckWithServiceAuthzWithServerAuthz) { + MockApiManagerEnvironmentWithLog env; std::unique_ptr config = - Config::Create(&env, kServiceNameConfig, server_config); + Config::Create(&env, kServiceConfigWithAuthz, kServerConfigWithAuthz); ASSERT_TRUE(config); ASSERT_EQ(config->GetFirebaseServer(), "https://myfirebaseserver.com/"); } -TEST(Config, TestEmptyFirebaseServerCheck) { +TEST(Config, TestFirebaseServerCheckWithoutServiceAuthzWithoutServerAuthz) { MockApiManagerEnvironmentWithLog env; - static const char server_config[] = R"( -service_control_config { - check_aggregator_config { - cache_entries: 1000 - flush_interval_ms: 10 - response_expiration_ms: 20 - } - report_aggregator_config { - cache_entries: 1020 - flush_interval_ms: 15 - } + std::unique_ptr config = Config::Create( + &env, kServiceConfigWithoutAuthz, kServerConfigWithoutAuthz); + ASSERT_TRUE(config); + + ASSERT_EQ(config->GetFirebaseServer(), ""); } -)"; + +TEST(Config, TestFirebaseServerCheckWithoutServiceConfigWithServerConfig) { + MockApiManagerEnvironmentWithLog env; std::unique_ptr config = - Config::Create(&env, kServiceNameConfig, server_config); + Config::Create(&env, kServiceConfigWithoutAuthz, kServerConfigWithAuthz); ASSERT_TRUE(config); - ASSERT_TRUE(config->GetFirebaseServer().empty()); + ASSERT_EQ(config->GetFirebaseServer(), "https://myfirebaseserver.com/"); } } // namespace diff --git a/contrib/endpoints/src/api_manager/context/service_context.h b/contrib/endpoints/src/api_manager/context/service_context.h index ea215567136..61524813157 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.h +++ b/contrib/endpoints/src/api_manager/context/service_context.h @@ -67,10 +67,7 @@ class ServiceContext { bool IsRulesCheckEnabled() const { return RequireAuth() && service().apis_size() > 0 && - (!config_->GetFirebaseServer().empty() || - (service().has_experimental() && - service().experimental().has_authorization() && - service().experimental().authorization().has_provider())); + !config_->GetFirebaseServer().empty(); } auth::Certs &certs() { return certs_; } From f90136c99f2ec8f77ee895b8ccb013fa8a2c0e04 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Mon, 13 Feb 2017 15:44:33 -0800 Subject: [PATCH 08/15] Unit tests for check security rules. (#75) * Unit tests for check security rules. * format * Address review comments. * Fix typos --- contrib/endpoints/src/api_manager/BUILD | 15 + .../src/api_manager/check_security_rules.cc | 12 +- .../api_manager/check_security_rules_test.cc | 461 ++++++++++++++++++ 3 files changed, 482 insertions(+), 6 deletions(-) create mode 100644 contrib/endpoints/src/api_manager/check_security_rules_test.cc diff --git a/contrib/endpoints/src/api_manager/BUILD b/contrib/endpoints/src/api_manager/BUILD index ffe4f060b7c..39e85eacac5 100644 --- a/contrib/endpoints/src/api_manager/BUILD +++ b/contrib/endpoints/src/api_manager/BUILD @@ -279,3 +279,18 @@ cc_test( "//external:googletest_main", ], ) + +cc_test( + name = "check_security_rules_test", + size = "small", + srcs = [ + "check_security_rules_test.cc", + "mock_request.h", + ], + linkstatic = 1, + deps = [ + ":api_manager", + ":mock_api_manager_environment", + "//external:googletest_main", + ], +) diff --git a/contrib/endpoints/src/api_manager/check_security_rules.cc b/contrib/endpoints/src/api_manager/check_security_rules.cc index 45fa30ae7bd..845bc4cf6bb 100644 --- a/contrib/endpoints/src/api_manager/check_security_rules.cc +++ b/contrib/endpoints/src/api_manager/check_security_rules.cc @@ -43,10 +43,10 @@ const char kFirebaseCreateMethod[] = "create"; const char kFirebaseGetMethod[] = "get"; const char kFirebaseDeleteMethod[] = "delete"; const char kFirebaseUpdateMethod[] = "update"; -const char kV1[] = "v1/"; +const char kV1[] = "/v1"; const char kTestQuery[] = ":test?alt=json"; -const char kProjects[] = "projects/"; -const char kReleases[] = "/releases/"; +const char kProjects[] = "/projects"; +const char kReleases[] = "/releases"; const char kRulesetName[] = "rulesetName"; const char kTestResults[] = "testResults"; const char kState[] = "state"; @@ -71,14 +71,14 @@ std::string GetReleaseName(const context::RequestContext &context) { std::string GetRulesetTestUri(const context::RequestContext &context, const std::string &ruleset_id) { - return context.service_context()->config()->GetFirebaseServer() + kV1 + + return context.service_context()->config()->GetFirebaseServer() + kV1 + "/" + ruleset_id + kTestQuery; } std::string GetReleaseUrl(const context::RequestContext &context) { return context.service_context()->config()->GetFirebaseServer() + kV1 + - kProjects + context.service_context()->project_id() + kReleases + - GetReleaseName(context); + kProjects + "/" + context.service_context()->project_id() + kReleases + + "/" + GetReleaseName(context); } std::string GetOperation(const std::string &httpMethod) { diff --git a/contrib/endpoints/src/api_manager/check_security_rules_test.cc b/contrib/endpoints/src/api_manager/check_security_rules_test.cc new file mode 100644 index 00000000000..97c555bc4df --- /dev/null +++ b/contrib/endpoints/src/api_manager/check_security_rules_test.cc @@ -0,0 +1,461 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// + +#include "contrib/endpoints/src/api_manager/check_security_rules.h" +#include "contrib/endpoints/src/api_manager/context/request_context.h" +#include "contrib/endpoints/src/api_manager/context/service_context.h" +#include "contrib/endpoints/src/api_manager/mock_api_manager_environment.h" +#include "contrib/endpoints/src/api_manager/mock_request.h" + +using ::testing::_; +using ::testing::AllOf; +using ::testing::Invoke; +using ::testing::Property; +using ::testing::Return; +using ::testing::StrCaseEq; +using ::testing::StrEq; +using ::testing::StrNe; + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { + +namespace { +// Service name to be used. +const char kServiceName[] = R"( +name: "myfirebaseapp.appspot.com" +)"; + +// Bad service name that will result in bad release name +const char kBadServiceName[] = R"( +name: "badService.appspot.com" +)"; + +// Producer project Id used to create the release URL +const char kProducerProjectId[] = R"( +producer_project_id: "myfirebaseapp" +)"; + +// APIs definition. +const char kApis[] = R"( +apis { + name: "Bookstore" + version: "v1" + methods { + name: "ListShelves" + request_type_url: "types.googleapis.com/google.protobuf.Empty" + response_type_url: "types.googleapis.com/Bookstore.ListShelvesResponse" + } + methods { + name: "ListBooks" + request_type_url: "types.googleapis.com/google.protobuf.Empty" + response_type_url: "types.googleapis.com/Bookstore.ListBooksResponse" + } + methods { + name: "CreateBook" + request_type_url: "types.googleapis.com/Bookstore.CreateBookRequest" + response_type_url: "types.googleapis.com/Bookstore.Book" + } +})"; + +// Authentication part of service config +const char kAuthentication[] = R"( +authentication { + providers { + id: "issuer1" + issuer: "https://issuer1.com" + } + providers { + id: "issuer2" + issuer: "https://issuer2.com" + jwks_uri: "https://issuer2.com/pubkey" + } + rules { + selector: "ListShelves" + requirements { + provider_id: "issuer1" + } + requirements { + provider_id: "issuer2" + } + } +})"; + +// Http part of service config +const char kHttp[] = R"( +http { + rules { + selector: "ListShelves" + get: "/ListShelves" + } +} +control { + environment: "http://127.0.0.1:8081" +})"; + +// Jwt payload to be used +const char kJwtEmailPayload[] = + R"({"iss":"https://accounts.google.com","iat":1486575396,"exp":1486578996,"aud":"https://myfirebaseapp.appspot.com","sub":"113424383671131376652","email_verified":true,"azp":"limin-429@appspot.gserviceaccount.com","email":"limin-429@appspot.gserviceaccount.com"})"; + +// Firebase Server config +static const char kServerConfig[] = R"( +api_check_security_rules_config { + firebase_server: "https://myfirebaseserver.com" +})"; + +// The response to GetRelease call to firebase server. +static const char kRelease[] = R"( +{ + "name": "projects/myfirebaseapp/releases/myfirebaseapp.appspot.com:v1", + "rulesetName": "projects/myfirebaseapp/rulesets/99045fc0-a5e4-47e2-a665-f88593594b6b", + "createTime": "2017-01-10T16:52:27.764111Z", + "updateTime": "2017-01-10T16:52:27.764111Z" +})"; + +// Error response for GetRelease on bad release name +static const char kReleaseError[] = R"( +{ + "error": { + "code": 404, + "message": "Requested entity was not found.", + "status": "NOT_FOUND", + } +})"; + +// TestRuleset returns Failure which means unauthorized access. +static const char kTestResultFailure[] = R"( +{ + "testResults": [ + { + "state": "FAILURE" + } + ] +} +)"; + +// TestRuleset call to Firebase response on success. +static const char kTestResultSuccess[] = R"( +{ + "testResults": [ + { + "state": "SUCCESS" + } + ] +} +)"; + +// Get a server configuration that has auth disabled. This should disable +// security rules check by default. +std::pair GetConfigWithAuthForceDisabled() { + std::string service_config = + std::string(kServiceName) + kApis + kAuthentication + kHttp; + const char server_config[] = R"( +api_authentication_config { + force_disable: true +} +api_check_security_rules_config { + firebase_server: "https://myfirebaseserver.com" +} +)"; + return std::make_pair(service_config, server_config); +} + +// Get service configuration with no authentication member field. This will +// disable auth and will also disable security rules check. +std::pair GetConfigWithNoAuth() { + std::string service_config = std::string(kServiceName) + kApis + kHttp; + return std::make_pair(service_config, std::string(kServerConfig)); +} + +// Get Service configuration with no apis. This will result in the version field +// not present and should disable security rules check. +std::pair GetConfigWithoutApis() { + std::string service_config = + std::string(kServiceName) + kAuthentication + kHttp; + + return std::make_pair(service_config, std::string(kServerConfig)); +} + +// There is no firebase server configuration. +std::pair GetConfigWithoutServer() { + std::string service_config = + std::string(kServiceName) + kApis + kAuthentication + kHttp; + return std::make_pair(service_config, ""); +} + +// Get a valid configuration. This will enable security check rules. +std::pair GetValidConfig() { + std::string service_config = + std::string(kServiceName) + kApis + kAuthentication + kHttp; + return std::make_pair(service_config, kServerConfig); +} + +// This test class is parameterized and creates Config object based on the +// service and server configuration provided. +class CheckDisableSecurityRulesTest + : public ::testing::TestWithParam> { + public: + void SetUp() { + std::unique_ptr env( + new ::testing::NiceMock()); + MockApiManagerEnvironment *raw_env = env.get(); + + std::string service_config; + std::string server_config; + + std::tie(service_config, server_config) = GetParam(); + std::unique_ptr config = + Config::Create(raw_env, service_config, server_config); + + ASSERT_TRUE(config != nullptr); + + service_context_ = std::make_shared( + std::move(env), std::move(config)); + + ASSERT_TRUE(service_context_.get() != nullptr); + + std::unique_ptr request( + new ::testing::NiceMock()); + + request_context_ = std::make_shared( + service_context_, std::move(request)); + EXPECT_CALL(*raw_env, DoRunHTTPRequest(_)).Times(0); + } + + std::shared_ptr request_context_; + std::shared_ptr service_context_; +}; + +// Paramterized test that will check for various configurations that will +// disable auth. +TEST_P(CheckDisableSecurityRulesTest, CheckAuthzDisabled) { + CheckSecurityRules(request_context_, + [](Status status) { ASSERT_TRUE(status.ok()); }); +} + +// Invoke the tests on CheckDisableSecurityRulesTest with various parameters. +INSTANTIATE_TEST_CASE_P(ConfigToDisableFirebaseRulesCheck, + CheckDisableSecurityRulesTest, + testing::Values(GetConfigWithNoAuth(), + GetConfigWithoutApis(), + GetConfigWithoutServer())); + +// Class that sets up the required objects to test various scenarios. +class CheckSecurityRulesTest : public ::testing::Test { + public: + void SetUp(std::string service_config, std::string server_config) { + std::unique_ptr env( + new ::testing::NiceMock()); + raw_env_ = env.get(); + + std::unique_ptr config = + Config::Create(raw_env_, service_config, server_config); + ASSERT_TRUE(config != nullptr); + + service_context_ = std::make_shared( + std::move(env), std::move(config)); + + ASSERT_TRUE(service_context_.get() != nullptr); + + std::unique_ptr request( + new ::testing::NiceMock()); + raw_request_ = request.get(); + + ON_CALL(*raw_request_, GetRequestHTTPMethod()) + .WillByDefault(Return(std::string("GET"))); + + ON_CALL(*raw_request_, GetRequestPath()) + .WillByDefault(Return(std::string("/ListShelves"))); + + request_context_ = std::make_shared( + service_context_, std::move(request)); + release_url_ = + "https://myfirebaseserver.com/v1/projects/myfirebaseapp/" + "releases/myfirebaseapp.appspot.com:v1"; + + ruleset_test_url_ = + "https://myfirebaseserver.com/v1" + "/projects/myfirebaseapp/rulesets/99045fc0-a5e4-47e2-a665-f88593594b6b" + ":test?alt=json"; + } + + MockApiManagerEnvironment *raw_env_; + MockRequest *raw_request_; + std::shared_ptr request_context_; + std::shared_ptr service_context_; + std::string release_url_; + std::string ruleset_test_url_; +}; + +// If the release name is bad, then check the following: +// 1. Ensure that GetRuleset request is inovked on bad release name. +// 2. In this case return Status with NOT_FOUND +// 3. Ensure that there are no more HTTP calls made to firbase TestRuleset +TEST_F(CheckSecurityRulesTest, CheckAuthzFailGetRelease) { + std::string service_config = std::string(kBadServiceName) + + kProducerProjectId + kApis + kAuthentication + + kHttp; + std::string server_config = kServerConfig; + SetUp(service_config, server_config); + + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(AllOf( + Property(&HTTPRequest::url, StrNe(release_url_)), + Property(&HTTPRequest::method, StrCaseEq("GET"))))) + .WillOnce(Invoke([](HTTPRequest *req) { + + std::map empty; + std::string body(kReleaseError); + req->OnComplete( + Status(Code::NOT_FOUND, "Requested entity was not found"), + std::move(empty), std::move(body)); + + })); + + EXPECT_CALL(*raw_env_, + DoRunHTTPRequest( + AllOf(Property(&HTTPRequest::url, StrEq(ruleset_test_url_)), + Property(&HTTPRequest::method, StrCaseEq("POST"))))) + .Times(0); + + auto ptr = this; + CheckSecurityRules(request_context_, + [ptr](Status status) { ASSERT_TRUE(!status.ok()); }); +} + +// Check that the right status is returned when TestRuleset completes with an +// error. This is modelled as an internal error. +// 1. Ensure that GetRelease is invoked on the correct release url and correct +// method. +// 2. The mock will respond with the ruleset Id. +// 3. Ensure that TestRuleset is invoked on the the righ URl and method. +// 4. In this case, mock will return an INTERNAL ERROR. +// 5. Make sure that status is not OK in this case. +TEST_F(CheckSecurityRulesTest, CheckAuthzFailTestRuleset) { + std::string service_config = std::string(kServiceName) + kProducerProjectId + + kApis + kAuthentication + kHttp; + std::string server_config = kServerConfig; + SetUp(service_config, server_config); + + request_context_->set_auth_claims(kJwtEmailPayload); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(AllOf( + Property(&HTTPRequest::url, StrEq(release_url_)), + Property(&HTTPRequest::method, StrCaseEq("GET"))))) + .WillOnce(Invoke([](HTTPRequest *req) { + + std::map empty; + std::string body(kRelease); + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + + })); + + EXPECT_CALL(*raw_env_, + DoRunHTTPRequest( + AllOf(Property(&HTTPRequest::url, StrEq(ruleset_test_url_)), + Property(&HTTPRequest::method, StrCaseEq("POST"))))) + .WillOnce(Invoke([](HTTPRequest *req) { + std::map empty; + std::string body; + req->OnComplete(Status(Code::INTERNAL, "Cannot talk to server"), + std::move(empty), std::move(body)); + })); + + CheckSecurityRules(request_context_, [](Status status) { + ASSERT_TRUE(status.CanonicalCode() == Code::INTERNAL); + }); +} + +// Check behavior when TestResultset return a "FAILURE" message. +// 1. Ensure GetRelease is invoked properly and in this case mock responds with +// the ruelset Id. +// 2. Ensure that the TestResultset is invoked correctly and respond wit ha +// Status::OK but with Failure body. +// 3. Asser that the final status returned is PERMISSION DENIED. +TEST_F(CheckSecurityRulesTest, CheckAuthzFailWithTestResultFailure) { + std::string service_config = std::string(kServiceName) + kProducerProjectId + + kApis + kAuthentication + kHttp; + std::string server_config = kServerConfig; + SetUp(service_config, server_config); + + request_context_->set_auth_claims(kJwtEmailPayload); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(AllOf( + Property(&HTTPRequest::url, StrEq(release_url_)), + Property(&HTTPRequest::method, StrCaseEq("GET"))))) + .WillOnce(Invoke([](HTTPRequest *req) { + + std::map empty; + std::string body(kRelease); + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + + })); + + EXPECT_CALL(*raw_env_, + DoRunHTTPRequest( + AllOf(Property(&HTTPRequest::url, StrEq(ruleset_test_url_)), + Property(&HTTPRequest::method, StrCaseEq("POST"))))) + .WillOnce(Invoke([](HTTPRequest *req) { + std::map empty; + std::string body = kTestResultFailure; + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + })); + + CheckSecurityRules(request_context_, [](Status status) { + ASSERT_TRUE(status.CanonicalCode() == Code::PERMISSION_DENIED); + }); +} + +// Check for success case. +// 1. Ensure GetRelease is invoked properly and in this case mock responds with +// the ruelset Id. +// 2. Ensure that the TestResultset is invoked correctly and respond wit ha +// Status::OK but with SUCCESS body. +// 3. Asser that the final status returned is OK. +TEST_F(CheckSecurityRulesTest, CheckAuthzSuccess) { + std::string service_config = std::string(kServiceName) + kProducerProjectId + + kApis + kAuthentication + kHttp; + std::string server_config = kServerConfig; + SetUp(service_config, server_config); + + request_context_->set_auth_claims(kJwtEmailPayload); + EXPECT_CALL(*raw_env_, DoRunHTTPRequest(AllOf( + Property(&HTTPRequest::url, StrEq(release_url_)), + Property(&HTTPRequest::method, StrCaseEq("GET"))))) + .WillOnce(Invoke([](HTTPRequest *req) { + + std::map empty; + std::string body(kRelease); + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + + })); + + EXPECT_CALL(*raw_env_, + DoRunHTTPRequest( + AllOf(Property(&HTTPRequest::url, StrEq(ruleset_test_url_)), + Property(&HTTPRequest::method, StrCaseEq("POST"))))) + .WillOnce(Invoke([](HTTPRequest *req) { + std::map empty; + std::string body = kTestResultSuccess; + req->OnComplete(Status::OK, std::move(empty), std::move(body)); + })); + + CheckSecurityRules(request_context_, + [](Status status) { ASSERT_TRUE(status.ok()); }); +} +} +} // namespace api_manager +} // namespace google From df4b7e43aad8e8a25d83606389c41d06ecef32ac Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Thu, 2 Mar 2017 14:39:33 -0800 Subject: [PATCH 09/15] Merge from master to firebase (#143) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect (#38) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect * Fixed style. * Rebase Envoy (#41) * Update prototype to use iptables (#42) * Rebase to fixed Envoy (#43) * Handle HEAD request. (#34) * Handle HEAD request. * Try with GET if HEAD fails. * Address comments. * Format file. * Expose bazel target (#48) * Try again (#49) * Integrate with mixer client. (#55) * Integrate with mixer client. * Restore repositories.bzl back. * Add originIp and originHost attributes. (#56) * Add uuid-dev dependency in README.md (#45) * Extract originIp and OriginHost. (#57) * Extract originIp and OriginHost. * Make header x-forwarded-host const. * Update buckets for UI. (#58) * Update buckets for UI. * Only update time_distribution. * Add targetService attribute. (#59) * Use envoy new access_log handler for sending Report. (#60) * use access_log handler. * Not to use Loggable base class. * Update to the latest envoy with #396. (#61) * Fix tclap dependency fetching error (#62) * Integrate mixer client directly with envoy. (#66) * Integrate mixer client directly with envoy. * Send response header in Report. * rename filter name from esp to mixer. * add README. * Add release binary script. (#68) * Push tar.gz to GCS (#69) * Push tar.gz to GCS * Rename envoy_esp * Remove mixer_client from api_manager. (#72) * Update mixer client SHA. (#74) * Update readme. (#73) * Adds Jenkinsfile and updates release-binary to create a SHA. (#71) * Adds Jenkinsfile and update release-binary * Update Jenkinsfile and gitignore * Fixes typo and use normal build Node * Uses default bazel config * Using batch mode * Update bazel memory settings * Do not use Jenkins bazel env * Set .bazelrc for postsubmit * Update grpc and protobuf (#70) * protobuf v3.2.0 * grpc v1.1.1 * Align auth lib with grpc 1.1.1 * Add sourceService. (#78) * Add script to build docker image. (#77) * Add script to build docker image. * Add start_envoy for docker image. * Use official attribute names (#80) * Use official attribute names * fix format * Creates a KEY for mixer client dep. Updates release-binary (#79) * Updated mixer repo to use a key for commit * release-binary skip build if file exists. * Update src/envoy/mixer/README. (#82) * Fix src/envoy/mixer/README.md (#85) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Not to use api_key if its service is not actived. (#109) * Update envoy and add c-ares (#107) * Update envoy and add c-ares depedencies * Update release script with debug and normal binary * remove debug ls * formatting * Send StatusCode Attributes to Mixer. (#110) * Add send_attribute filter. (#115) * Add send_attribute filter. * Fix format * rename variable serialized_attributes_ * Address the comments. * Fail request if api_key is not valid. (#116) * Fail request if api_key is not valid. * Format code. * Update comments. * Address comment. * Rename response.http.code (#125) * Send headers as string map. (#129) * Send headers as string map. * Remove origin.ip and origin.host. * Fix format * unify bazel's docker build targets with other istio repos (#127) * update base debug docker image reference (#133) * Update postsubmit to create docker images (#132) * Adding config release for bazel build (#135) * Fix mixer client crash. (#136) * Get mixerclient with response parsing. (#138) * Update nghttp2 to sync with envoy (#140) * Fix src/envoy/mixer/README.md * Update nghttp2 to sync with envoy * update * fix typo --- .bazelrc.jenkins | 8 + .gitignore | 2 + .travis.yml | 4 + Jenkinsfile | 65 ++ WORKSPACE | 37 +- contrib/endpoints/repositories.bzl | 17 +- .../auth/lib/auth_jwt_validator.cc | 47 +- .../src/api_manager/auth/lib/auth_token.cc | 3 +- .../src/api_manager/auth/lib/grpc_internals.h | 176 +---- .../endpoints/src/api_manager/context/BUILD | 1 - .../api_manager/context/request_context.cc | 3 +- .../api_manager/context/service_context.cc | 15 +- .../endpoints/src/api_manager/mixer/mixer.cc | 204 ----- .../endpoints/src/api_manager/mixer/mixer.h | 77 -- .../src/api_manager/proto/server_config.proto | 9 - .../api_manager/server_config_proto_test.cc | 6 - .../api_manager/service_control/aggregated.cc | 10 +- .../src/api_manager/service_control/proto.cc | 15 +- docker/BUILD | 11 + repositories.bzl | 4 +- script/release-binary | 58 ++ .../mixer/BUILD => script/release-docker | 56 +- src/envoy/mixer/BUILD | 106 +++ src/envoy/mixer/README.md | 99 +++ src/envoy/mixer/envoy.conf.template | 137 ++++ src/envoy/mixer/forward_attribute_filter.cc | 114 +++ src/envoy/mixer/http_control.cc | 194 +++++ src/envoy/mixer/http_control.h | 64 ++ src/envoy/mixer/http_filter.cc | 277 +++++++ src/envoy/mixer/proxy_docker.bzl | 9 + .../BUILD => mixer/repositories.bzl} | 31 +- src/envoy/mixer/start_envoy | 67 ++ src/envoy/mixer/string_map.proto | 23 + src/envoy/mixer/utils.cc | 50 ++ src/envoy/mixer/utils.h | 39 + src/envoy/prototype/README.md | 73 -- src/envoy/prototype/api_manager_env.cc | 219 ------ src/envoy/prototype/api_manager_env.h | 33 - src/envoy/prototype/api_manager_filter.cc | 346 -------- .../prototype/dummy_api_manager_cluster.py | 91 --- src/envoy/prototype/envoy-esp.conf | 89 --- .../prototype/generic_service_config.json | 736 ------------------ src/envoy/prototype/server_config.pb.txt | 6 - src/envoy/repositories.bzl | 129 ++- 44 files changed, 1591 insertions(+), 2169 deletions(-) create mode 100644 .bazelrc.jenkins create mode 100644 Jenkinsfile delete mode 100644 contrib/endpoints/src/api_manager/mixer/mixer.cc delete mode 100644 contrib/endpoints/src/api_manager/mixer/mixer.h create mode 100644 docker/BUILD create mode 100755 script/release-binary rename contrib/endpoints/src/api_manager/mixer/BUILD => script/release-docker (51%) mode change 100644 => 100755 create mode 100644 src/envoy/mixer/BUILD create mode 100644 src/envoy/mixer/README.md create mode 100644 src/envoy/mixer/envoy.conf.template create mode 100644 src/envoy/mixer/forward_attribute_filter.cc create mode 100644 src/envoy/mixer/http_control.cc create mode 100644 src/envoy/mixer/http_control.h create mode 100644 src/envoy/mixer/http_filter.cc create mode 100644 src/envoy/mixer/proxy_docker.bzl rename src/envoy/{prototype/BUILD => mixer/repositories.bzl} (57%) create mode 100755 src/envoy/mixer/start_envoy create mode 100644 src/envoy/mixer/string_map.proto create mode 100644 src/envoy/mixer/utils.cc create mode 100644 src/envoy/mixer/utils.h delete mode 100644 src/envoy/prototype/README.md delete mode 100644 src/envoy/prototype/api_manager_env.cc delete mode 100644 src/envoy/prototype/api_manager_env.h delete mode 100644 src/envoy/prototype/api_manager_filter.cc delete mode 100644 src/envoy/prototype/dummy_api_manager_cluster.py delete mode 100644 src/envoy/prototype/envoy-esp.conf delete mode 100644 src/envoy/prototype/generic_service_config.json delete mode 100644 src/envoy/prototype/server_config.pb.txt diff --git a/.bazelrc.jenkins b/.bazelrc.jenkins new file mode 100644 index 00000000000..664f7b13380 --- /dev/null +++ b/.bazelrc.jenkins @@ -0,0 +1,8 @@ +# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. +startup --host_jvm_args=-Xmx8192m +startup --host_jvm_args=-Xms8192m +startup --batch + +# This is so we understand failures better +build --verbose_failures + diff --git a/.gitignore b/.gitignore index a6ef824c1f8..469486c8f36 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /bazel-* +.idea/* +*.iml diff --git a/.travis.yml b/.travis.yml index d8559616e6d..88be42c285b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ sudo: required dist: xenial +branches: + except: + - stable + lang: go go: diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000000..9c59a146e50 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,65 @@ +#!groovy + +@Library('testutils') + +import org.istio.testutils.Utilities +import org.istio.testutils.GitUtilities +import org.istio.testutils.Bazel + +// Utilities shared amongst modules +def gitUtils = new GitUtilities() +def utils = new Utilities() +def bazel = new Bazel() + +mainFlow(utils) { + pullRequest(utils) { + node { + gitUtils.initialize() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') + } + + if (utils.runStage('PRESUBMIT')) { + presubmit(gitUtils, bazel) + } + if (utils.runStage('POSTSUBMIT')) { + postsubmit(gitUtils, bazel, utils) + } + } +} + +def presubmit(gitUtils, bazel) { + buildNode(gitUtils) { + stage('Code Check') { + sh('script/check-style') + } + bazel.updateBazelRc() + stage('Bazel Fetch') { + bazel.fetch('-k //...') + } + stage('Bazel Build') { + bazel.build('//...') + } + stage('Bazel Tests') { + bazel.test('//...') + } + stage('Push Test Binary') { + sh 'script/release-binary' + } + } +} + +def postsubmit(gitUtils, bazel, utils) { + buildNode(gitUtils) { + bazel.updateBazelRc() + stage('Push Binary') { + sh 'script/release-binary' + } + stage('Docker Push') { + def images = 'proxy,proxy_debug' + def tags = "${gitUtils.GIT_SHORT_SHA},\$(date +%Y-%m-%d-%H.%M.%S),latest" + utils.publishDockerImages(images, tags, 'release') + } + } +} \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index 95dcdde5cdc..ddfa05d578b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -31,16 +31,20 @@ googletest_repositories() load( "//contrib/endpoints:repositories.bzl", "grpc_repositories", - "mixer_client_repositories", "servicecontrol_client_repositories", ) grpc_repositories() -mixer_client_repositories() - servicecontrol_client_repositories() +load( + "//src/envoy/mixer:repositories.bzl", + "mixer_client_repositories", +) + +mixer_client_repositories() + # Workaround for Bazel > 0.4.0 since it needs newer protobuf.bzl from: # https://github.com/google/protobuf/pull/2246 # Do not use this git_repository for anything else than protobuf.bzl @@ -65,3 +69,30 @@ load( ) envoy_repositories() + +new_http_archive( + name = "docker_ubuntu", + build_file_content = """ +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") +docker_build( + name = "xenial", + tars = ["xenial/ubuntu-xenial-core-cloudimg-amd64-root.tar.gz"], + visibility = ["//visibility:public"], +) +""", + sha256 = "de31e6fcb843068965de5945c11a6f86399be5e4208c7299fb7311634fb41943", + strip_prefix = "docker-brew-ubuntu-core-e406914e5f648003dfe8329b512c30c9ad0d2f9c", + type = "zip", + url = "https://codeload.github.com/tianon/docker-brew-ubuntu-core/zip/e406914e5f648003dfe8329b512c30c9ad0d2f9c", +) + + +DEBUG_BASE_IMAGE_SHA="3f57ae2aceef79e4000fb07ec850bbf4bce811e6f81dc8cfd970e16cdf33e622" + +# See github.com/istio/manager/blob/master/docker/debug/build-and-publish-debug-image.sh +# for instructions on how to re-build and publish this base image layer. +http_file( + name = "ubuntu_xenial_debug", + url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug-" + DEBUG_BASE_IMAGE_SHA + ".tar.gz", + sha256 = DEBUG_BASE_IMAGE_SHA, +) diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index bed9315a267..c77a3f72f58 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -163,7 +163,7 @@ def grpc_repositories(bind=True): native.git_repository( name = "grpc_git", - commit = "d28417c856366df704200f544e72d31056931bce", + commit = "bb3edafea245a9780cc4c10f0b58da21e8193f38", # v1.1.1 remote = "https://github.com/grpc/grpc.git", ) @@ -190,7 +190,7 @@ def grpc_repositories(bind=True): native.bind( name = "grpc_lib", - actual = "@grpc_git//:grpc++_reflection", + actual = "@grpc_git//:grpc++_codegen_proto", ) def googleapis_repositories(protobuf_repo="@protobuf_git//", bind=True): @@ -335,16 +335,3 @@ def servicecontrol_client_repositories(bind=True): name = "servicecontrol_client", actual = "@servicecontrol_client_git//:service_control_client_lib", ) - -def mixer_client_repositories(bind=True): - native.git_repository( - name = "mixerclient_git", - commit = "80e450a5126960e8e6337c3631cf2ef984038eab", - remote = "https://github.com/istio/mixerclient.git", - ) - - if bind: - native.bind( - name = "mixer_client_lib", - actual = "@mixerclient_git//:mixer_client_lib", - ) diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc index 2df9a843b71..962d3980bea 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc @@ -150,6 +150,8 @@ class JwtValidatorImpl : public JwtValidator { RSA *rsa_; EVP_PKEY *pkey_; EVP_MD_CTX *md_ctx_; + + grpc_exec_ctx exec_ctx_; }; // Gets EVP_MD mapped from an alg (algorithm string). @@ -159,12 +161,12 @@ const EVP_MD *EvpMdFromAlg(const char *alg); size_t HashSizeFromAlg(const char *alg); // Parses str into grpc_json object. Does not own buffer. -grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, - gpr_slice *buffer); +grpc_json *DecodeBase64AndParseJson(grpc_exec_ctx *exec_ctx, const char *str, + size_t len, gpr_slice *buffer); // Gets BIGNUM from b64 string, used for extracting pkey from jwk. // Result owned by rsa_. -BIGNUM *BigNumFromBase64String(const char *b64); +BIGNUM *BigNumFromBase64String(grpc_exec_ctx *exec_ctx, const char *b64); } // namespace @@ -185,7 +187,8 @@ JwtValidatorImpl::JwtValidatorImpl(const char *jwt, size_t jwt_len) x509_(nullptr), rsa_(nullptr), pkey_(nullptr), - md_ctx_(nullptr) { + md_ctx_(nullptr), + exec_ctx_(GRPC_EXEC_CTX_INIT) { header_buffer_ = gpr_empty_slice(); signed_buffer_ = gpr_empty_slice(); sig_buffer_ = gpr_empty_slice(); @@ -204,7 +207,7 @@ JwtValidatorImpl::~JwtValidatorImpl() { grpc_json_destroy(pkey_json_); } if (claims_ != nullptr) { - grpc_jwt_claims_destroy(claims_); + grpc_jwt_claims_destroy(&exec_ctx_, claims_); } if (!GPR_SLICE_IS_EMPTY(header_buffer_)) { gpr_slice_unref(header_buffer_); @@ -304,7 +307,8 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { if (dot == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } - header_json_ = DecodeBase64AndParseJson(cur, dot - cur, &header_buffer_); + header_json_ = + DecodeBase64AndParseJson(&exec_ctx_, cur, dot - cur, &header_buffer_); CreateJoseHeader(); if (header_ == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; @@ -323,7 +327,7 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { // case, and it is owned by claims_ for successful case. gpr_slice claims_buffer = gpr_empty_slice(); grpc_json *claims_json = - DecodeBase64AndParseJson(cur, dot - cur, &claims_buffer); + DecodeBase64AndParseJson(&exec_ctx_, cur, dot - cur, &claims_buffer); if (claims_json == nullptr) { if (!GPR_SLICE_IS_EMPTY(claims_buffer)) { gpr_slice_unref(claims_buffer); @@ -332,10 +336,13 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { } UpdateAudience(claims_json); // Takes ownershp of claims_json and claims_buffer. - claims_ = grpc_jwt_claims_from_json(claims_json, claims_buffer); - if (claims_ == nullptr) { + claims_ = grpc_jwt_claims_from_json(&exec_ctx_, claims_json, claims_buffer); + + // issuer is mandatory. grpc_jwt_claims_issuer checks if claims_ is nullptr. + if (grpc_jwt_claims_issuer(claims_) == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } + // Check timestamp. // Passing in its own audience to skip audience check. // Audience check should be done by the caller. @@ -354,8 +361,8 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { return GRPC_JWT_VERIFIER_BAD_FORMAT; } cur = dot + 1; - sig_buffer_ = - grpc_base64_decode_with_len(cur, jwt_len - signed_jwt_len - 1, 1); + sig_buffer_ = grpc_base64_decode_with_len(&exec_ctx_, cur, + jwt_len - signed_jwt_len - 1, 1); if (GPR_SLICE_IS_EMPTY(sig_buffer_)) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } @@ -576,9 +583,11 @@ bool JwtValidatorImpl::ExtractPubkeyFromJwk(const grpc_json *jkey) { } const char *rsa_n = GetStringValue(jkey, "n"); - rsa_->n = rsa_n == nullptr ? nullptr : BigNumFromBase64String(rsa_n); + rsa_->n = + rsa_n == nullptr ? nullptr : BigNumFromBase64String(&exec_ctx_, rsa_n); const char *rsa_e = GetStringValue(jkey, "e"); - rsa_->e = rsa_e == nullptr ? nullptr : BigNumFromBase64String(rsa_e); + rsa_->e = + rsa_e == nullptr ? nullptr : BigNumFromBase64String(&exec_ctx_, rsa_e); if (rsa_->e == nullptr || rsa_->n == nullptr) { gpr_log(GPR_ERROR, "Missing RSA public key field."); @@ -651,7 +660,7 @@ grpc_jwt_verifier_status JwtValidatorImpl::VerifyHsSignature(const char *pkey, const EVP_MD *md = EvpMdFromAlg(header_->alg); GPR_ASSERT(md != nullptr); // Checked before. - pkey_buffer_ = grpc_base64_decode_with_len(pkey, pkey_len, 1); + pkey_buffer_ = grpc_base64_decode_with_len(&exec_ctx_, pkey, pkey_len, 1); if (GPR_SLICE_IS_EMPTY(pkey_buffer_)) { gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; @@ -742,11 +751,11 @@ size_t HashSizeFromAlg(const char *alg) { } } -grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, - gpr_slice *buffer) { +grpc_json *DecodeBase64AndParseJson(grpc_exec_ctx *exec_ctx, const char *str, + size_t len, gpr_slice *buffer) { grpc_json *json; - *buffer = grpc_base64_decode_with_len(str, len, 1); + *buffer = grpc_base64_decode_with_len(exec_ctx, str, len, 1); if (GPR_SLICE_IS_EMPTY(*buffer)) { gpr_log(GPR_ERROR, "Invalid base64."); return nullptr; @@ -760,12 +769,12 @@ grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, return json; } -BIGNUM *BigNumFromBase64String(const char *b64) { +BIGNUM *BigNumFromBase64String(grpc_exec_ctx *exec_ctx, const char *b64) { BIGNUM *result = nullptr; gpr_slice bin; if (b64 == nullptr) return nullptr; - bin = grpc_base64_decode(b64, 1); + bin = grpc_base64_decode(exec_ctx, b64, 1); if (GPR_SLICE_IS_EMPTY(bin)) { gpr_log(GPR_ERROR, "Invalid base64 for big num."); return nullptr; diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc index 72b53790158..85f06ca0417 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc @@ -200,7 +200,8 @@ char *GenerateJwtClaim(const char *issuer, const char *subject, } char *GenerateSignatueHs256(const char *data, const char *key) { - gpr_slice key_buffer = grpc_base64_decode(key, 1); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_slice key_buffer = grpc_base64_decode(&exec_ctx, key, 1); if (GPR_SLICE_IS_EMPTY(key_buffer)) { gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); return nullptr; diff --git a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h index 66074a12776..cdf58840654 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h +++ b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h @@ -15,11 +15,6 @@ #ifndef API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ #define API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ -// This header file contains definitions for all grpc internal -// dependencies. The code that depends on grpc internals should -// include this file instead of including the original headers -// in grpc. - // This header file is for internal use only since it declares grpc // internals that auth depends on. A public header file should not // include any internal grpc header files. @@ -30,171 +25,12 @@ extern "C" { -#include -#include - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/json/json_common.h -////////////////////////////////////////////////////// - -/* The various json types. */ -typedef enum { - GRPC_JSON_OBJECT, - GRPC_JSON_ARRAY, - GRPC_JSON_STRING, - GRPC_JSON_NUMBER, - GRPC_JSON_TRUE, - GRPC_JSON_FALSE, - GRPC_JSON_NULL, - GRPC_JSON_TOP_LEVEL -} grpc_json_type; - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/json/json.h -////////////////////////////////////////////////////// - -/* A tree-like structure to hold json values. The key and value pointers - * are not owned by it. - */ -typedef struct grpc_json { - struct grpc_json *next; - struct grpc_json *prev; - struct grpc_json *child; - struct grpc_json *parent; - - grpc_json_type type; - const char *key; - const char *value; -} grpc_json; - -/* The next two functions are going to parse the input string, and - * modify it in the process, in order to use its space to store - * all of the keys and values for the returned object tree. - * - * They assume UTF-8 input stream, and will output UTF-8 encoded - * strings in the tree. The input stream's UTF-8 isn't validated, - * as in, what you input is what you get as an output. - * - * All the keys and values in the grpc_json objects will be strings - * pointing at your input buffer. - * - * Delete the allocated tree afterward using grpc_json_destroy(). - */ -grpc_json *grpc_json_parse_string_with_len(char *input, size_t size); -grpc_json *grpc_json_parse_string(char *input); - -/* Use these to create or delete a grpc_json object. - * Deletion is recursive. We will not attempt to free any of the strings - * in any of the objects of that tree. - */ -grpc_json *grpc_json_create(grpc_json_type type); -void grpc_json_destroy(grpc_json *json); - -/* This function will create a new string using gpr_realloc, and will - * deserialize the grpc_json tree into it. It'll be zero-terminated, - * but will be allocated in chunks of 256 bytes. - * - * The indent parameter controls the way the output is formatted. - * If indent is 0, then newlines will be suppressed as well, and the - * output will be condensed at its maximum. - */ -char *grpc_json_dump_to_string(grpc_json *json, int indent); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/security/base64 -////////////////////////////////////////////////////// - -/* Encodes data using base64. It is the caller's responsability to free - the returned char * using gpr_free. Returns nullptr on nullptr input. */ -char *grpc_base64_encode(const void *data, size_t data_size, int url_safe, - int multiline); - -/* Decodes data according to the base64 specification. Returns an empty - slice in case of failure. */ -gpr_slice grpc_base64_decode(const char *b64, int url_safe); - -/* Same as above except that the length is provided by the caller. */ -gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, - int url_safe); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/auth/security/json_key.h -////////////////////////////////////////////////////// - -/* --- auth_json_key parsing. --- */ - -typedef struct { - const char *type; - char *private_key_id; - char *client_id; - char *client_email; - RSA *private_key; -} grpc_auth_json_key; - -/* Creates a json_key object from string. Returns an invalid object if a parsing - error has been encountered. */ -grpc_auth_json_key grpc_auth_json_key_create_from_string( - const char *json_string); - -/* Destructs the object. */ -void grpc_auth_json_key_destruct(grpc_auth_json_key *json_key); - -/* Caller is responsible for calling gpr_free on the returned value. May return - nullptr on invalid input. The scope parameter may be nullptr. */ -char *grpc_jwt_encode_and_sign(const grpc_auth_json_key *json_key, - const char *audience, - gpr_timespec token_lifetime, const char *scope); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/support/string.h -////////////////////////////////////////////////////// - -/* Minimum buffer size for calling ltoa */ -#define GPR_LTOA_MIN_BUFSIZE (3 * sizeof(long)) - -/* Convert a long to a string in base 10; returns the length of the - output string (or 0 on failure). - output must be at least GPR_LTOA_MIN_BUFSIZE bytes long. */ -int gpr_ltoa(long value, char *output); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/security/jwt_verifier.h -////////////////////////////////////////////////////// - -/* --- grpc_jwt_verifier_status. --- */ - -typedef enum { - GRPC_JWT_VERIFIER_OK = 0, - GRPC_JWT_VERIFIER_BAD_SIGNATURE, - GRPC_JWT_VERIFIER_BAD_FORMAT, - GRPC_JWT_VERIFIER_BAD_AUDIENCE, - GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR, - GRPC_JWT_VERIFIER_TIME_CONSTRAINT_FAILURE, - GRPC_JWT_VERIFIER_GENERIC_ERROR -} grpc_jwt_verifier_status; - -const char *grpc_jwt_verifier_status_to_string(grpc_jwt_verifier_status status); - -/* --- grpc_jwt_claims. --- */ - -typedef struct grpc_jwt_claims grpc_jwt_claims; - -void grpc_jwt_claims_destroy(grpc_jwt_claims *claims); - -/* Returns the whole JSON tree of the claims. */ -const grpc_json *grpc_jwt_claims_json(const grpc_jwt_claims *claims); - -/* Access to registered claims in https://tools.ietf.org/html/rfc7519#page-9 */ -const char *grpc_jwt_claims_subject(const grpc_jwt_claims *claims); -const char *grpc_jwt_claims_issuer(const grpc_jwt_claims *claims); -const char *grpc_jwt_claims_audience(const grpc_jwt_claims *claims); -gpr_timespec grpc_jwt_claims_expires_at(const grpc_jwt_claims *claims); - -/* --- TESTING ONLY exposed functions. --- */ - -grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer); -grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, - const char *audience); +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_common.h" +#include "src/core/lib/security/credentials/jwt/json_token.h" +#include "src/core/lib/security/credentials/jwt/jwt_verifier.h" +#include "src/core/lib/security/util/b64.h" +#include "src/core/lib/support/string.h" } #endif // API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ diff --git a/contrib/endpoints/src/api_manager/context/BUILD b/contrib/endpoints/src/api_manager/context/BUILD index e158c13fc92..59365102318 100644 --- a/contrib/endpoints/src/api_manager/context/BUILD +++ b/contrib/endpoints/src/api_manager/context/BUILD @@ -39,7 +39,6 @@ cc_library( "//contrib/endpoints/src/api_manager/auth", "//contrib/endpoints/src/api_manager/auth:service_account_token", "//contrib/endpoints/src/api_manager/cloud_trace", - "//contrib/endpoints/src/api_manager/mixer", "//contrib/endpoints/src/api_manager/service_control", "//contrib/endpoints/src/api_manager/utils", "//external:cc_wkt_protos", diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index 59cc7c3d53f..75bcb177f9d 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -165,7 +165,8 @@ void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { info->operation_name = kUnrecognizedOperation; } info->operation_id = operation_id_; - if (check_response_info_.is_api_key_valid) { + if (check_response_info_.is_api_key_valid && + check_response_info_.service_is_activated) { info->api_key = api_key_; } info->producer_project_id = service_context()->project_id(); diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index 0e99c247676..28ce9271f80 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -15,7 +15,6 @@ //////////////////////////////////////////////////////////////////////////////// // #include "contrib/endpoints/src/api_manager/context/service_context.h" -#include "contrib/endpoints/src/api_manager/mixer/mixer.h" #include "contrib/endpoints/src/api_manager/service_control/aggregated.h" @@ -105,16 +104,10 @@ const std::string& ServiceContext::project_id() const { } std::unique_ptr ServiceContext::CreateInterface() { - if (config_->server_config() && - config_->server_config()->has_mixer_options()) { - return std::unique_ptr( - mixer::Mixer::Create(env_.get(), config_.get())); - } else { - return std::unique_ptr( - service_control::Aggregated::Create( - config_->service(), config_->server_config(), env_.get(), - &service_account_token_)); - } + return std::unique_ptr( + service_control::Aggregated::Create(config_->service(), + config_->server_config(), env_.get(), + &service_account_token_)); } std::unique_ptr diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc deleted file mode 100644 index 53837edaf67..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "contrib/endpoints/src/api_manager/mixer/mixer.h" - -using ::google::api_manager::utils::Status; -using ::istio::mixer_client::Attributes; - -namespace google { -namespace api_manager { -namespace mixer { -namespace { - -const std::string kProxyPeerID = "Istio/Proxy"; -const std::string kEnvNameTargetService = "TARGET_SERVICE"; - -const std::string kAttrNameServiceName = "serviceName"; -const std::string kAttrNamePeerId = "peerId"; -const std::string kAttrNameOperationId = "operationId"; -const std::string kAttrNameOperationName = "operationName"; -const std::string kAttrNameApiKey = "apiKey"; -const std::string kAttrNameResponseCode = "responseCode"; -const std::string kAttrNameURL = "url"; -const std::string kAttrNameLocation = "location"; -const std::string kAttrNameApiName = "apiName"; -const std::string kAttrNameApiVersion = "apiVersion"; -const std::string kAttrNameApiMethod = "apiMethod"; -const std::string kAttrNameRequestSize = "requestSize"; -const std::string kAttrNameResponseSize = "responseSize"; -const std::string kAttrNameLogMessage = "logMessage"; -const std::string kAttrNameResponseTime = "responseTime"; -const std::string kAttrNameOriginIp = "originIp"; -const std::string kAttrNameOriginHost = "originHost"; -const std::string kAttrNameTargetService = "targetService"; - -Attributes::Value StringValue(const std::string& str) { - Attributes::Value v; - v.type = Attributes::Value::STRING; - v.str_v = str; - return v; -} - -Attributes::Value Int64Value(int64_t value) { - Attributes::Value v; - v.type = Attributes::Value::INT64; - v.value.int64_v = value; - return v; -} - -} // namespace - -Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) - : env_(env), config_(config) {} - -Mixer::~Mixer() {} - -Status Mixer::Init() { - ::istio::mixer_client::MixerClientOptions options; - options.mixer_server = - config_->server_config()->mixer_options().mixer_server(); - mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); - auto target_service = getenv(kEnvNameTargetService.c_str()); - if (target_service) { - target_service_ = target_service; - } - return Status::OK; -} - -Status Mixer::Close() { return Status::OK; } - -void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, - ::istio::mixer_client::Attributes* attr) { - attr->attributes[kAttrNameServiceName] = StringValue(config_->service_name()); - attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); - - if (!info.operation_id.empty()) { - attr->attributes[kAttrNameOperationId] = StringValue(info.operation_id); - } - if (!info.operation_name.empty()) { - attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); - } - if (!info.api_key.empty()) { - attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); - } - if (!info.client_ip.empty()) { - attr->attributes[kAttrNameOriginIp] = StringValue(info.client_ip); - } - if (!info.client_host.empty()) { - attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); - } - if (!target_service_.empty()) { - attr->attributes[kAttrNameTargetService] = StringValue(target_service_); - } -} - -void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, - ::istio::mixer_client::Attributes* attr) { - FillCommonAttributes(info, attr); -} - -void Mixer::FillReportAttributes(const service_control::ReportRequestInfo& info, - ::istio::mixer_client::Attributes* attr) { - FillCommonAttributes(info, attr); - - if (!info.url.empty()) { - attr->attributes[kAttrNameURL] = StringValue(info.url); - } - if (!info.location.empty()) { - attr->attributes[kAttrNameLocation] = StringValue(info.location); - } - - if (!info.api_name.empty()) { - attr->attributes[kAttrNameApiName] = StringValue(info.api_name); - } - if (!info.api_version.empty()) { - attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); - } - if (!info.api_method.empty()) { - attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); - } - - if (!info.log_message.empty()) { - attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); - } - - attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); - if (info.request_size >= 0) { - attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); - } - if (info.response_size >= 0) { - attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); - } - - if (info.latency.request_time_ms >= 0) { - attr->attributes[kAttrNameResponseTime] = - Int64Value(info.latency.request_time_ms); - } -} - -Status Mixer::Report(const service_control::ReportRequestInfo& info) { - ::istio::mixer_client::Attributes attributes; - FillReportAttributes(info, &attributes); - env_->LogInfo("Send Report: "); - env_->LogInfo(attributes.DebugString()); - mixer_client_->Report( - attributes, [this](const ::google::protobuf::util::Status& status) { - if (status.ok()) { - env_->LogInfo("Report response: OK"); - } else { - env_->LogError(std::string("Failed to call Mixer::report, Error: ") + - status.ToString()); - } - }); - return Status::OK; -} - -void Mixer::Check( - const service_control::CheckRequestInfo& info, - cloud_trace::CloudTraceSpan* parent_span, - std::function - on_done) { - ::istio::mixer_client::Attributes attributes; - FillCheckAttributes(info, &attributes); - env_->LogInfo("Send Check: "); - env_->LogInfo(attributes.DebugString()); - mixer_client_->Check( - attributes, - [this, on_done](const ::google::protobuf::util::Status& status) { - if (status.ok()) { - env_->LogInfo("Check response: OK"); - } else { - env_->LogError(std::string("Failed to call Mixer::check, Error: ") + - status.ToString()); - } - service_control::CheckResponseInfo info; - on_done(Status(status.error_code(), status.error_message(), - Status::SERVICE_CONTROL), - info); - }); -} - -Status Mixer::GetStatistics(service_control::Statistics* esp_stat) const { - return Status::OK; -} - -service_control::Interface* Mixer::Create(ApiManagerEnvInterface* env, - const Config* config) { - return new Mixer(env, config); -} - -} // namespace mixer -} // namespace api_manager -} // namespace google diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h deleted file mode 100644 index 81276e1227d..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef API_MANAGER_MIXER_MIXER_H_ -#define API_MANAGER_MIXER_MIXER_H_ - -#include "contrib/endpoints/include/api_manager/env_interface.h" -#include "contrib/endpoints/src/api_manager/config.h" -#include "contrib/endpoints/src/api_manager/service_control/interface.h" -#include "include/client.h" - -namespace google { -namespace api_manager { -namespace mixer { - -// This implementation uses service-control-client-cxx module. -class Mixer : public service_control::Interface { - public: - static service_control::Interface* Create(ApiManagerEnvInterface* env, - const Config* config); - - virtual ~Mixer(); - - virtual utils::Status Report(const service_control::ReportRequestInfo& info); - - virtual void Check( - const service_control::CheckRequestInfo& info, - cloud_trace::CloudTraceSpan* parent_span, - std::function - on_done); - - virtual utils::Status Init(); - virtual utils::Status Close(); - - virtual utils::Status GetStatistics(service_control::Statistics* stat) const; - - private: - // The constructor. - Mixer(ApiManagerEnvInterface* env, const Config* config); - - // Fill common attributes for both check and report. - void FillCommonAttributes(const service_control::OperationInfo& info, - ::istio::mixer_client::Attributes* attr); - // Fill attributes for check. - void FillCheckAttributes(const service_control::CheckRequestInfo& info, - ::istio::mixer_client::Attributes* attr); - // Fill attributes for report. - void FillReportAttributes(const service_control::ReportRequestInfo& info, - ::istio::mixer_client::Attributes* attr); - - // The Api Manager environment interface. - ApiManagerEnvInterface* env_; - // The config. - const Config* config_; - // The mixer client - std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; - // Target service - std::string target_service_; -}; - -} // namespace mixer -} // namespace api_manager -} // namespace google - -#endif // API_MANAGER_MIXER_MIXER_H_ diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index 9e325c3f1a0..cbee58f8da8 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -35,10 +35,6 @@ message ServerConfig { // Server config used for API authentication ApiAuthenticationConfig api_authentication_config = 5; - // Mixer option flag. If not present, default to use service_control. When - // Envoy/esp talks to Mixer, has to specify this field. - MixerOptions mixer_options = 6; - // Server config used for API authorization via Firebase Rules. ApiCheckSecurityRulesConfig api_check_security_rules_config = 7; @@ -152,11 +148,6 @@ message ApiCheckSecurityRulesConfig { string firebase_server = 1; } -message MixerOptions { - // For envoy, it is the cluster name for mixer server. - string mixer_server = 1; -} - message Experimental { // Disable timed printouts of ESP status to the error log. bool disable_log_status = 1; diff --git a/contrib/endpoints/src/api_manager/server_config_proto_test.cc b/contrib/endpoints/src/api_manager/server_config_proto_test.cc index b39309727f1..80c62e05471 100644 --- a/contrib/endpoints/src/api_manager/server_config_proto_test.cc +++ b/contrib/endpoints/src/api_manager/server_config_proto_test.cc @@ -76,9 +76,6 @@ experimental { disable_log_status: false } -mixer_options { - mixer_server: "mixer_server" -} )"; TEST(ServerConfigProto, ServerConfigFromString) { @@ -125,9 +122,6 @@ TEST(ServerConfigProto, ServerConfigFromString) { // Check disable_log_status EXPECT_EQ(false, server_config.experimental().disable_log_status()); - - // Check mixer options - EXPECT_EQ("mixer_server", server_config.mixer_options().mixer_server()); } TEST(ServerConfigProto, ValidateSampleServerConfig) { diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.cc b/contrib/endpoints/src/api_manager/service_control/aggregated.cc index d2754d69dbd..e59a69dc8cc 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated.cc +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.cc @@ -287,8 +287,14 @@ void Aggregated::Check( if (status.ok()) { Status status = Proto::ConvertCheckResponse( *response, service_control_proto_.service_name(), &response_info); - // If allow_unregistered_calls is true, it is always OK to proceed. - if (allow_unregistered_calls) { + // If server replied with either invalid api_key or not activated service, + // the request is rejected even allow_unregistered_calls is true. Most + // likely, users provide a wrong api key. By failing the request, the + // users will be notified with the error and have chance to correct it. + // Otherwise, the Report call will fail. It is very hard to notice and + // debug the Report failure. + if (allow_unregistered_calls && response_info.is_api_key_valid && + response_info.service_is_activated) { on_done(Status::OK, response_info); } else { on_done(status, response_info); diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 3e7eba64d21..694f299d22c 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -956,14 +956,13 @@ Status Proto::FillReportRequest(const ReportRequestInfo& info, } } - // Not to send consumer metrics for following cases: - // 1) api_key is not provided, or - // 2) the service is not activated for the consumer project, - bool send_consumer_metric = true; - if (info.api_key.empty() || - !info.check_response_info.service_is_activated) { - send_consumer_metric = false; - } + // Not to send consumer metrics if api_key is empty. + // api_key is empty in one of following cases: + // 1) api_key is not provided, + // 2) api_key is invalid determined by the server from the Check call. + // 3) the service is not activated for the consumer project. + bool send_consumer_metric = !info.api_key.empty(); + // Populate all metrics. for (auto it = metrics_.begin(), end = metrics_.end(); it != end; it++) { const SupportedMetric* m = *it; diff --git a/docker/BUILD b/docker/BUILD new file mode 100644 index 00000000000..6e017ee6064 --- /dev/null +++ b/docker/BUILD @@ -0,0 +1,11 @@ +# Add aliases for envoy specific targets so that the docker build +# targets are consistent for all istio repos, e.g. //docker: +alias( + name = "proxy", + actual = "//src/envoy/mixer:proxy", +) + +alias( + name = "proxy_debug", + actual = "//src/envoy/mixer:proxy_debug", +) diff --git a/repositories.bzl b/repositories.bzl index 8764aa24d32..31fe15ffb63 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -36,7 +36,7 @@ def boringssl_repositories(bind=True): def protobuf_repositories(bind=True): native.git_repository( name = "protobuf_git", - commit = "a428e42072765993ff674fda72863c9f1aa2d268", # v3.1.0 + commit = "593e917c176b5bc5aafa57bf9f6030d749d91cd5", # v3.2.0 remote = "https://github.com/google/protobuf.git", ) @@ -68,7 +68,7 @@ def protobuf_repositories(bind=True): native.bind( name = "protobuf_clib", - actual = "@protobuf_git//:protobuf_lite", + actual = "@protobuf_git//:protoc_lib", ) diff --git a/script/release-binary b/script/release-binary new file mode 100755 index 00000000000..0ff4dd63f91 --- /dev/null +++ b/script/release-binary @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +set -ex + +# Make sure to this script on x86_64 Ubuntu Xenial +UBUNTU_RELEASE="$(lsb_release -c -s)" +[[ "${UBUNTU_RELEASE}" == 'xenial' ]] || { echo 'must run on Ubuntu Xenial'; exit 1; } + +# The bucket name to store proxy binary +DST="gs://istio-build/proxy" + +# The proxy binary name. +SHA="$(git rev-parse --verify HEAD)" +BINARY_NAME="envoy-alpha-${SHA}.tar.gz" +SHA256_NAME="envoy-alpha-${SHA}.sha256" + +# If binary already exists skip. +gsutil stat "${DST}/${BINARY_NAME}" \ + && { echo 'Binary already exists'; exit 0; } \ + || echo 'Building a new binary.' + +# Build the release binary +bazel build --config=release //src/envoy/mixer:envoy_tar +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" + +# Copy it to the bucket. +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" + +# Build the debug binary +BINARY_NAME="envoy-debug-${SHA}.tar.gz" +SHA256_NAME="envoy-debug-${SHA}.sha256" +bazel build -c dbg //src/envoy/mixer:envoy_tar +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" + +# Copy it to the bucket. +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" diff --git a/contrib/endpoints/src/api_manager/mixer/BUILD b/script/release-docker old mode 100644 new mode 100755 similarity index 51% rename from contrib/endpoints/src/api_manager/mixer/BUILD rename to script/release-docker index affe09c7ddd..d21caffee55 --- a/contrib/endpoints/src/api_manager/mixer/BUILD +++ b/script/release-docker @@ -1,3 +1,5 @@ +#!/bin/bash +# # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,29 +16,31 @@ # ################################################################################ # -package(default_visibility = ["//contrib/endpoints/src/api_manager:__subpackages__"]) - -cc_library( - name = "mixer", - srcs = [ - "mixer.cc", - ], - hdrs = [ - "mixer.h", - ], - linkopts = select({ - "//:darwin": [], - "//conditions:default": [ - "-lm", - "-luuid", - ], - }), - deps = [ - "//contrib/endpoints/src/api_manager:impl_headers", - "//contrib/endpoints/src/api_manager/service_control", - "//contrib/endpoints/src/api_manager/utils", - "//external:grpc++", - "//external:mixer_client_lib", - "//external:protobuf", - ], -) +set -ex + +# This docker image can be used as: +# +# docker run IMAGE -a backend_address -p PORT -m MIXER_SERVER +# + +PROJECT=istio-testing +IMAGE_PREFIX="gcr.io/${PROJECT}/proxy" + +DATE_PART=$(date +"%Y%m%d") +SHA_PART=$(git show -q HEAD --pretty=format:%h) +DOCKER_TAG="${DATE_PART}${SHA_PART}" + +IMAGE_NAME="${IMAGE_PREFIX}:${DOCKER_TAG}" + +gcloud docker --authorize-only + +bazel run --config=release //src/envoy/mixer:proxy "${IMAGE_NAME}" + +gcloud docker -- push "${IMAGE_NAME}" + +IMAGE_LATEST="${IMAGE_PREFIX}:latest" + +docker tag -f "${IMAGE_NAME}" "${IMAGE_LATEST}" + +gcloud docker -- push "${IMAGE_LATEST}" + diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD new file mode 100644 index 00000000000..132bc364a1d --- /dev/null +++ b/src/envoy/mixer/BUILD @@ -0,0 +1,106 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + + +load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("//src/envoy/mixer:proxy_docker.bzl", "proxy_docker_build") +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") + +cc_proto_library( + name = "string_map_proto", + srcs = ["string_map.proto"], + default_runtime = "//external:protobuf", + protoc = "//external:protoc", + visibility = ["//visibility:public"], +) + +cc_library( + name = "filter_lib", + srcs = [ + "forward_attribute_filter.cc", + "http_control.cc", + "http_control.h", + "http_filter.cc", + "utils.cc", + "utils.h", + ], + deps = [ + ":string_map_proto", + "//external:mixer_client_lib", + "@envoy_git//:envoy-common", + ], + alwayslink = 1, +) + +cc_binary( + name = "envoy", + linkstatic = 1, + deps = [ + ":filter_lib", + "@envoy_git//:envoy-main", + ], +) + +pkg_tar( + name = "envoy_tar", + extension = "tar.gz", + files = [":envoy"], + mode = "0755", + package_dir = "/usr/local/bin/", +) + +pkg_tar( + name = "start_envoy_tar", + extension = "tar.gz", + files = ["start_envoy"], + mode = "0755", + package_dir = "/usr/local/bin/", +) + +pkg_tar( + name = "config_tar", + extension = "tar.gz", + files = ["envoy.conf.template"], + mode = "0666", + package_dir = "/etc/opt/proxy", + tags = ["manual"], +) + +proxy_docker_build( + images = [ + {"name": "proxy", "base": "@docker_ubuntu//:xenial"}, + {"name": "proxy_debug", "base": "@ubuntu_xenial_debug//file"}, + ], + entrypoint = [ + "/usr/local/bin/start_envoy", + "-e", + "/usr/local/bin/envoy", + "-c", + "/etc/opt/proxy/envoy.conf", + "-t", + "/etc/opt/proxy/envoy.conf.template", + ], + ports = ["9090"], + repository = "istio", + tags = ["manual"], + tars = [ + ":config_tar", + ":envoy_tar", + ":start_envoy_tar", + ], + visibility = ["//visibility:public"], +) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md new file mode 100644 index 00000000000..d7d39f30e03 --- /dev/null +++ b/src/envoy/mixer/README.md @@ -0,0 +1,99 @@ + +This Proxy will use Envoy and talk to Mixer server. + +## Build Mixer server + +* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: + +``` + cd $(ISTIO)/mixer + bazel build ...:all +``` + +## Build Envoy proxy + +* Build target envoy: + +``` + bazel build //src/envoy/mixer:envoy +``` + +## How to run it + +* Start mixer server. In mixer folder run: + +``` + bazel-bin/cmd/server/mixs server + --globalConfigFile testdata/globalconfig.yml + --serviceConfigFile testdata/serviceconfig.yml --logtostderr +``` + + The server will run at port 9091 + +* Start backend Echo server. + +``` + cd test/backend/echo + go run echo.go +``` + +* Start Envoy proxy, run + +``` + src/envoy/mixer/start_envoy +``` + +* Then issue HTTP request to proxy. + +``` + curl http://localhost:9090/echo -d "hello world" +``` + +## How to configurate HTTP filters + +This module has two HTTP filters: +1. mixer filter: intercept all HTTP requests, call the mixer. +2. forward_attribute filter: Forward attributes to the upstream istio/proxy. + +### *mixer* filter: + +This filter will intercept all HTTP requests and call Mixer. Here is its config: + +``` + "filters": [ + "type": "both", + "name": "mixer", + "config": { + "mixer_server": "${MIXER_SERVER}", + "attributes" : { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + } + } +``` + +Notes: +* mixer_server is required +* attributes: these attributes will be send to the mixer + +### *forward_attribute* HTTP filter: + +This filer will forward attributes to the upstream istio/proxy. + +``` + "filters": [ + "type": "decoder", + "name": "forward_attribute", + "config": { + "attributes": { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + } + } +``` + +Notes: +* attributes: these attributes will be forwarded to the upstream istio/proxy. + + + diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template new file mode 100644 index 00000000000..c5377632388 --- /dev/null +++ b/src/envoy/mixer/envoy.conf.template @@ -0,0 +1,137 @@ +{ + "listeners": [ + { + "port": ${PORT}, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service1" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "both", + "name": "mixer", + "config": { + "mixer_server": "${MIXER_SERVER}", + "attributes": { + "target.uid": "POD222", + "target.namespace": "XYZ222" + } + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + }, + { + "port": 7070, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service2" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "decoder", + "name": "forward_attribute", + "config": { + "attributes": { + "source.uid": "POD11", + "source.namespace": "XYZ11" + } + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + } + ], + "admin": { + "access_log_path": "/dev/stdout", + "port": 9001 + }, + "cluster_manager": { + "clusters": [ + { + "name": "service1", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://${BACKEND}" + } + ] + }, + { + "name": "service2", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:9090" + } + ] + } + ] + } +} diff --git a/src/envoy/mixer/forward_attribute_filter.cc b/src/envoy/mixer/forward_attribute_filter.cc new file mode 100644 index 00000000000..08d58916f89 --- /dev/null +++ b/src/envoy/mixer/forward_attribute_filter.cc @@ -0,0 +1,114 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "precompiled/precompiled.h" + +#include "common/common/base64.h" +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "common/http/utility.h" +#include "envoy/server/instance.h" +#include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/utils.h" + +namespace Http { +namespace ForwardAttribute { +namespace { + +// The Json object name to specify attributes which will be forwarded +// to the upstream istio proxy. +const std::string kJsonNameAttributes("attributes"); + +} // namespace + +class Config : public Logger::Loggable { + private: + std::string attributes_; + + public: + Config(const Json::Object& config) { + Utils::StringMap attributes = + Utils::ExtractStringMap(config, kJsonNameAttributes); + if (!attributes.empty()) { + std::string serialized_str = Utils::SerializeStringMap(attributes); + attributes_ = + Base64::encode(serialized_str.c_str(), serialized_str.size()); + } + } + + const std::string& attributes() const { return attributes_; } +}; + +typedef std::shared_ptr ConfigPtr; + +class ForwardAttributeFilter : public Http::StreamDecoderFilter { + private: + ConfigPtr config_; + + public: + ForwardAttributeFilter(ConfigPtr config) : config_(config) {} + + FilterHeadersStatus decodeHeaders(HeaderMap& headers, + bool end_stream) override { + if (!config_->attributes().empty()) { + headers.addStatic(Utils::kIstioAttributeHeader, config_->attributes()); + } + return FilterHeadersStatus::Continue; + } + + FilterDataStatus decodeData(Buffer::Instance& data, + bool end_stream) override { + return FilterDataStatus::Continue; + } + + FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + return FilterTrailersStatus::Continue; + } + + void setDecoderFilterCallbacks( + StreamDecoderFilterCallbacks& callbacks) override {} +}; + +} // namespace ForwardAttribute +} // namespace Http + +namespace Server { +namespace Configuration { + +class ForwardAttributeConfig : public HttpFilterConfigFactory { + public: + HttpFilterFactoryCb tryCreateFilterFactory( + HttpFilterType type, const std::string& name, const Json::Object& config, + const std::string&, Server::Instance& server) override { + if (type != HttpFilterType::Decoder || name != "forward_attribute") { + return nullptr; + } + + Http::ForwardAttribute::ConfigPtr add_header_config( + new Http::ForwardAttribute::Config(config)); + return [add_header_config]( + Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::ForwardAttribute::ForwardAttributeFilter( + add_header_config)); + callbacks.addStreamDecoderFilter(Http::StreamDecoderFilterPtr(instance)); + }; + } +}; + +static RegisterHttpFilterConfigFactory register_; + +} // namespace Configuration +} // namespace server diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc new file mode 100644 index 00000000000..6e64659c2bb --- /dev/null +++ b/src/envoy/mixer/http_control.cc @@ -0,0 +1,194 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/http_control.h" + +#include "common/common/base64.h" +#include "common/common/utility.h" +#include "common/http/utility.h" + +#include "src/envoy/mixer/string_map.pb.h" +#include "src/envoy/mixer/utils.h" + +using ::google::protobuf::util::Status; +using ::istio::mixer_client::Attributes; +using ::istio::mixer_client::DoneFunc; + +namespace Http { +namespace Mixer { +namespace { + +// Define attribute names +const std::string kRequestPath = "request.path"; +const std::string kRequestHost = "request.host"; +const std::string kRequestSize = "request.size"; +const std::string kRequestTime = "request.time"; +const std::string kRequestHeaders = "request.headers"; + +const std::string kResponseHeaders = "response.headers"; +const std::string kResponseSize = "response.size"; +const std::string kResponseTime = "response.time"; +const std::string kResponseLatency = "response.latency"; +const std::string kResponseHttpCode = "response.http.code"; + +Attributes::Value StringValue(const std::string& str) { + Attributes::Value v; + v.type = Attributes::Value::STRING; + v.str_v = str; + return v; +} + +Attributes::Value StringMapValue( + std::map&& string_map) { + Attributes::Value v; + v.type = Attributes::Value::STRING_MAP; + v.string_map_v.swap(string_map); + return v; +} + +Attributes::Value Int64Value(int64_t value) { + Attributes::Value v; + v.type = Attributes::Value::INT64; + v.value.int64_v = value; + return v; +} + +Attributes::Value TimeValue( + std::chrono::time_point value) { + Attributes::Value v; + v.type = Attributes::Value::TIME; + v.time_v = value; + return v; +} + +Attributes::Value DurationValue(std::chrono::nanoseconds value) { + Attributes::Value v; + v.type = Attributes::Value::DURATION; + v.duration_nanos_v = value; + return v; +} + +void SetStringAttribute(const std::string& name, const std::string& value, + Attributes* attr) { + if (!value.empty()) { + attr->attributes[name] = StringValue(value); + } +} + +std::map ExtractHeaders(const HeaderMap& header_map) { + std::map headers; + header_map.iterate( + [](const HeaderEntry& header, void* context) { + std::map* header_map = + static_cast*>(context); + (*header_map)[header.key().c_str()] = header.value().c_str(); + }, + &headers); + return headers; +} + +void FillRequestHeaderAttributes(const HeaderMap& header_map, + Attributes* attr) { + SetStringAttribute(kRequestPath, header_map.Path()->value().c_str(), attr); + SetStringAttribute(kRequestHost, header_map.Host()->value().c_str(), attr); + attr->attributes[kRequestTime] = TimeValue(std::chrono::system_clock::now()); + attr->attributes[kRequestHeaders] = + StringMapValue(ExtractHeaders(header_map)); +} + +void FillResponseHeaderAttributes(const HeaderMap* header_map, + Attributes* attr) { + if (header_map) { + attr->attributes[kResponseHeaders] = + StringMapValue(ExtractHeaders(*header_map)); + } + attr->attributes[kResponseTime] = TimeValue(std::chrono::system_clock::now()); +} + +void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, + int check_status_code, Attributes* attr) { + if (info.bytesReceived() >= 0) { + attr->attributes[kRequestSize] = Int64Value(info.bytesReceived()); + } + if (info.bytesSent() >= 0) { + attr->attributes[kResponseSize] = Int64Value(info.bytesSent()); + } + + if (info.duration().count() > 0) { + attr->attributes[kResponseLatency] = DurationValue( + std::chrono::duration_cast(info.duration())); + } + + if (info.responseCode().valid()) { + attr->attributes[kResponseHttpCode] = + Int64Value(info.responseCode().value()); + } else { + attr->attributes[kResponseHttpCode] = Int64Value(check_status_code); + } +} + +} // namespace + +HttpControl::HttpControl(const std::string& mixer_server, + std::map&& attributes) + : config_attributes_(std::move(attributes)) { + ::istio::mixer_client::MixerClientOptions options; + options.mixer_server = mixer_server; + mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); +} + +void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { + // Extract attributes from x-istio-attributes header + const HeaderEntry* entry = header_map.get(Utils::kIstioAttributeHeader); + if (entry) { + ::istio::proxy::mixer::StringMap pb; + std::string str(entry->value().c_str(), entry->value().size()); + pb.ParseFromString(Base64::decode(str)); + for (const auto& it : pb.map()) { + SetStringAttribute(it.first, it.second, attr); + } + header_map.remove(Utils::kIstioAttributeHeader); + } + + FillRequestHeaderAttributes(header_map, attr); + + for (const auto& attribute : config_attributes_) { + SetStringAttribute(attribute.first, attribute.second, attr); + } +} + +void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, + DoneFunc on_done) { + FillCheckAttributes(headers, &request_data->attributes); + log().debug("Send Check: {}", request_data->attributes.DebugString()); + mixer_client_->Check(request_data->attributes, on_done); +} + +void HttpControl::Report(HttpRequestDataPtr request_data, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info, + int check_status, DoneFunc on_done) { + // Use all Check attributes for Report. + // Add additional Report attributes. + FillResponseHeaderAttributes(response_headers, &request_data->attributes); + + FillRequestInfoAttributes(request_info, check_status, + &request_data->attributes); + log().debug("Send Report: {}", request_data->attributes.DebugString()); + mixer_client_->Report(request_data->attributes, on_done); +} + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h new file mode 100644 index 00000000000..c5938cf369d --- /dev/null +++ b/src/envoy/mixer/http_control.h @@ -0,0 +1,64 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "envoy/http/access_log.h" +#include "include/client.h" + +namespace Http { +namespace Mixer { + +// Store data from Check to report +struct HttpRequestData { + ::istio::mixer_client::Attributes attributes; +}; +typedef std::shared_ptr HttpRequestDataPtr; + +// The mixer client class to control HTTP requests. +// It has Check() to validate if a request can be processed. +// At the end of request, call Report(). +class HttpControl final : public Logger::Loggable { + public: + // The constructor. + HttpControl(const std::string& mixer_server, + std::map&& attributes); + + // Make mixer check call. + void Check(HttpRequestDataPtr request_data, HeaderMap& headers, + ::istio::mixer_client::DoneFunc on_done); + + // Make mixer report call. + void Report(HttpRequestDataPtr request_data, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info, int check_status_code, + ::istio::mixer_client::DoneFunc on_done); + + private: + void FillCheckAttributes(HeaderMap& header_map, + ::istio::mixer_client::Attributes* attr); + + // The mixer client + std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // The attributes read from the config file. + std::map config_attributes_; +}; + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc new file mode 100644 index 00000000000..a990b0fd7e3 --- /dev/null +++ b/src/envoy/mixer/http_filter.cc @@ -0,0 +1,277 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "precompiled/precompiled.h" + +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "common/http/utility.h" +#include "envoy/server/instance.h" +#include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/http_control.h" +#include "src/envoy/mixer/utils.h" + +using ::google::protobuf::util::Status; +using StatusCode = ::google::protobuf::util::error::Code; +using ::istio::mixer_client::DoneFunc; + +namespace Http { +namespace Mixer { +namespace { + +// The Json object name for mixer-server. +const std::string kJsonNameMixerServer("mixer_server"); + +// The Json object name for static attributes. +const std::string kJsonNameMixerAttributes("attributes"); + +// Convert Status::code to HTTP code +int HttpCode(int code) { + // Map Canonical codes to HTTP status codes. This is based on the mapping + // defined by the protobuf http error space. + switch (code) { + case StatusCode::OK: + return 200; + case StatusCode::CANCELLED: + return 499; + case StatusCode::UNKNOWN: + return 500; + case StatusCode::INVALID_ARGUMENT: + return 400; + case StatusCode::DEADLINE_EXCEEDED: + return 504; + case StatusCode::NOT_FOUND: + return 404; + case StatusCode::ALREADY_EXISTS: + return 409; + case StatusCode::PERMISSION_DENIED: + return 403; + case StatusCode::RESOURCE_EXHAUSTED: + return 429; + case StatusCode::FAILED_PRECONDITION: + return 400; + case StatusCode::ABORTED: + return 409; + case StatusCode::OUT_OF_RANGE: + return 400; + case StatusCode::UNIMPLEMENTED: + return 501; + case StatusCode::INTERNAL: + return 500; + case StatusCode::UNAVAILABLE: + return 503; + case StatusCode::DATA_LOSS: + return 500; + case StatusCode::UNAUTHENTICATED: + return 401; + default: + return 500; + } +} + +} // namespace + +class Config : public Logger::Loggable { + private: + std::shared_ptr http_control_; + Upstream::ClusterManager& cm_; + + public: + Config(const Json::Object& config, Server::Instance& server) + : cm_(server.clusterManager()) { + std::string mixer_server; + if (config.hasObject(kJsonNameMixerServer)) { + mixer_server = config.getString(kJsonNameMixerServer); + } else { + log().error( + "mixer_server is required but not specified in the config: {}", + __func__); + } + + std::map attributes = + Utils::ExtractStringMap(config, kJsonNameMixerAttributes); + + http_control_ = + std::make_shared(mixer_server, std::move(attributes)); + log().debug("Called Mixer::Config constructor with mixer_server: ", + mixer_server); + } + + std::shared_ptr& http_control() { return http_control_; } +}; + +typedef std::shared_ptr ConfigPtr; + +class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { + private: + std::shared_ptr http_control_; + std::shared_ptr request_data_; + + enum State { NotStarted, Calling, Complete, Responded }; + State state_; + + StreamDecoderFilterCallbacks* decoder_callbacks_; + StreamEncoderFilterCallbacks* encoder_callbacks_; + + bool initiating_call_; + int check_status_code_; + + public: + Instance(ConfigPtr config) + : http_control_(config->http_control()), + state_(NotStarted), + initiating_call_(false), + check_status_code_(HttpCode(StatusCode::UNKNOWN)) { + Log().debug("Called Mixer::Instance : {}", __func__); + } + + // Jump thread; on_done will be called at the dispatcher thread. + DoneFunc wrapper(DoneFunc on_done) { + auto& dispatcher = decoder_callbacks_->dispatcher(); + return [&dispatcher, on_done](const Status& status) { + dispatcher.post([status, on_done]() { on_done(status); }); + }; + } + + FilterHeadersStatus decodeHeaders(HeaderMap& headers, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + state_ = Calling; + initiating_call_ = true; + request_data_ = std::make_shared(); + http_control_->Check( + request_data_, headers, + wrapper([this](const Status& status) { completeCheck(status); })); + initiating_call_ = false; + + if (state_ == Complete) { + return FilterHeadersStatus::Continue; + } + Log().debug("Called Mixer::Instance : {} Stop", __func__); + return FilterHeadersStatus::StopIteration; + } + + FilterDataStatus decodeData(Buffer::Instance& data, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {} ({}, {})", __func__, data.length(), + end_stream); + if (state_ == Calling) { + return FilterDataStatus::StopIterationAndBuffer; + } + return FilterDataStatus::Continue; + } + + FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + Log().debug("Called Mixer::Instance : {}", __func__); + if (state_ == Calling) { + return FilterTrailersStatus::StopIteration; + } + return FilterTrailersStatus::Continue; + } + void setDecoderFilterCallbacks( + StreamDecoderFilterCallbacks& callbacks) override { + Log().debug("Called Mixer::Instance : {}", __func__); + decoder_callbacks_ = &callbacks; + decoder_callbacks_->addResetStreamCallback( + [this]() { state_ = Responded; }); + } + void completeCheck(const Status& status) { + Log().debug("Called Mixer::Instance : check complete {}", + status.ToString()); + if (!status.ok() && state_ != Responded) { + state_ = Responded; + check_status_code_ = HttpCode(status.error_code()); + Utility::sendLocalReply(*decoder_callbacks_, Code(check_status_code_), + status.ToString()); + return; + } + state_ = Complete; + if (!initiating_call_) { + decoder_callbacks_->continueDecoding(); + } + } + + virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterHeadersStatus::Continue; + } + virtual FilterDataStatus encodeData(Buffer::Instance& data, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterDataStatus::Continue; + } + virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterTrailersStatus::Continue; + } + virtual void setEncoderFilterCallbacks( + StreamEncoderFilterCallbacks& callbacks) override { + Log().debug("Called Mixer::Instance : {}", __func__); + encoder_callbacks_ = &callbacks; + } + + virtual void log(const HeaderMap* request_headers, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info) override { + Log().debug("Called Mixer::Instance : {}", __func__); + // Make sure not to use any class members at the callback. + // The class may be gone when it is called. + // Log() is a static function so it is OK. + http_control_->Report(request_data_, response_headers, request_info, + check_status_code_, [](const Status& status) { + Log().debug("Report returns status: {}", + status.ToString()); + }); + } + + static spdlog::logger& Log() { + static spdlog::logger& instance = + Logger::Registry::getLog(Logger::Id::http); + return instance; + } +}; + +} // namespace Mixer +} // namespace Http + +namespace Server { +namespace Configuration { + +class MixerConfig : public HttpFilterConfigFactory { + public: + HttpFilterFactoryCb tryCreateFilterFactory( + HttpFilterType type, const std::string& name, const Json::Object& config, + const std::string&, Server::Instance& server) override { + if (type != HttpFilterType::Both || name != "mixer") { + return nullptr; + } + + Http::Mixer::ConfigPtr mixer_config( + new Http::Mixer::Config(config, server)); + return + [mixer_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::Mixer::Instance(mixer_config)); + callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); + callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); + }; + } +}; + +static RegisterHttpFilterConfigFactory register_; + +} // namespace Configuration +} // namespace server diff --git a/src/envoy/mixer/proxy_docker.bzl b/src/envoy/mixer/proxy_docker.bzl new file mode 100644 index 00000000000..8522729628f --- /dev/null +++ b/src/envoy/mixer/proxy_docker.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") + +def proxy_docker_build(images, **kwargs): + for image in images: + docker_build( + name = image['name'], + base = image['base'], + **kwargs + ) diff --git a/src/envoy/prototype/BUILD b/src/envoy/mixer/repositories.bzl similarity index 57% rename from src/envoy/prototype/BUILD rename to src/envoy/mixer/repositories.bzl index 258f2046cd7..2f665d71c4b 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/mixer/repositories.bzl @@ -15,22 +15,17 @@ ################################################################################ # -package(default_visibility = ["//visibility:public"]) +MIXER_CLIENT = "5b5745f29ac5a8babe79ada573defaa83f3bb9e7" -cc_binary( - name = "envoy_esp", - srcs = [ - "api_manager_filter.cc", - "api_manager_env.cc", - "api_manager_env.h", - ], - deps = [ - "//external:protobuf", - "//contrib/endpoints/include:api_manager", - "//contrib/endpoints/src/grpc/transcoding:transcoding", - "//external:servicecontrol", - "@envoy_git//:envoy-common", - "@envoy_git//:envoy-main" - ], - linkstatic=1, -) +def mixer_client_repositories(bind=True): + native.git_repository( + name = "mixerclient_git", + commit = MIXER_CLIENT, + remote = "https://github.com/istio/mixerclient.git", + ) + + if bind: + native.bind( + name = "mixer_client_lib", + actual = "@mixerclient_git//:mixer_client_lib", + ) diff --git a/src/envoy/mixer/start_envoy b/src/envoy/mixer/start_envoy new file mode 100755 index 00000000000..0a24b959ae2 --- /dev/null +++ b/src/envoy/mixer/start_envoy @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +function usage() { + [[ -n "${1}" ]] && echo "${1}" + + cat < "${CONFIG}" + +"${ENVOY}" -c "${CONFIG}" "${DEBUG}" + diff --git a/src/envoy/mixer/string_map.proto b/src/envoy/mixer/string_map.proto new file mode 100644 index 00000000000..bd9140c37d7 --- /dev/null +++ b/src/envoy/mixer/string_map.proto @@ -0,0 +1,23 @@ +// Copyright 2017 Istio Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.proxy.mixer; + +// A message with a map of string to string. It is used to serialize +// a string map. +message StringMap { + map map = 1; +} diff --git a/src/envoy/mixer/utils.cc b/src/envoy/mixer/utils.cc new file mode 100644 index 00000000000..6dc3a78356d --- /dev/null +++ b/src/envoy/mixer/utils.cc @@ -0,0 +1,50 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/utils.h" +#include "src/envoy/mixer/string_map.pb.h" + +namespace Http { +namespace Utils { + +const LowerCaseString kIstioAttributeHeader("x-istio-attributes"); + +StringMap ExtractStringMap(const Json::Object& json, const std::string& name) { + StringMap map; + if (json.hasObject(name)) { + Json::ObjectPtr json_obj = json.getObject(name); + Json::Object* raw_obj = json_obj.get(); + json_obj->iterate( + [&map, raw_obj](const std::string& key, const Json::Object&) -> bool { + map[key] = raw_obj->getString(key); + return true; + }); + } + return map; +} + +std::string SerializeStringMap(const StringMap& string_map) { + ::istio::proxy::mixer::StringMap pb; + ::google::protobuf::Map* map_pb = pb.mutable_map(); + for (const auto& it : string_map) { + (*map_pb)[it.first] = it.second; + } + std::string str; + pb.SerializeToString(&str); + return str; +} + +} // namespace Utils +} // namespace Http diff --git a/src/envoy/mixer/utils.h b/src/envoy/mixer/utils.h new file mode 100644 index 00000000000..9273a9a2d8b --- /dev/null +++ b/src/envoy/mixer/utils.h @@ -0,0 +1,39 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/http/headers.h" +#include "envoy/json/json_object.h" + +namespace Http { +namespace Utils { + +// The internal header to pass istio attributes. +extern const LowerCaseString kIstioAttributeHeader; + +// The string map. +typedef std::map StringMap; + +// Extracts name/value attributes from a json object. +StringMap ExtractStringMap(const Json::Object& json, const std::string& name); + +// Serialize a string map to string. +std::string SerializeStringMap(const StringMap& map); + +} // namespace Utils +} // namespace Http diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md deleted file mode 100644 index dc73f63f6c9..00000000000 --- a/src/envoy/prototype/README.md +++ /dev/null @@ -1,73 +0,0 @@ - -This Proxy will use Envoy and talk to Mixer server. - -## Install dependencies - -``` - apt-get install uuid-dev -``` - -## Build Mixer server - -* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: - -``` - cd $(ISTIO)/mixer - bazel build ...:all -``` - -## Build Envoy proxy - -* Build target envoy_esp: - -``` - bazel build //src/envoy/prototype:envoy_esp -``` - -## How to run it - -* Start mixer server. In mixer folder run: - -``` - bazel-bin/cmd/server/mixs server -``` - - The server will run at port 9091 - -* Start backend Echo server. - -``` - cd test/backend/echo - go run echo.go -``` - -* Modify your iptables: - -``` - sudo iptables -t nat -A OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 -``` - -Once you are done, you should remove this rule: - -``` - sudo iptables -t nat -D OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 -``` - - -* Start Envoy proxy, run - -``` - bazel-bin/src/envoy/prototype/envoy_esp -c src/envoy/prototype/envoy-esp.conf -``` - -* Then issue HTTP request to proxy. - -``` - curl http://localhost:9090/echo?key=API-KEY -d "hello world" -``` - -## How to add attributes or facts - -Now only some of attributes are passed to mixer. If you want to add more attributes, you can -modify this -[file](https://github.com/istio/proxy/blob/master/contrib/endpoints/src/api_manager/mixer/mixer.cc). diff --git a/src/envoy/prototype/api_manager_env.cc b/src/envoy/prototype/api_manager_env.cc deleted file mode 100644 index f8536bcc10a..00000000000 --- a/src/envoy/prototype/api_manager_env.cc +++ /dev/null @@ -1,219 +0,0 @@ -#include "api_manager_env.h" - -#include "common/http/headers.h" -#include "common/http/message_impl.h" -#include "envoy/event/timer.h" -#include "google/protobuf/stubs/status.h" -#include "source/common/grpc/common.h" - -using ::google::api_manager::utils::Status; -using ::google::protobuf::util::error::Code; - -namespace Http { -namespace ApiManager { - -void Http::ApiManager::Env::Log(LogLevel level, const char *message) { - switch (level) { - case LogLevel::DEBUG: - log().debug("{}", message); - break; - case LogLevel::INFO: - log().info("{}", message); - break; - case LogLevel::WARNING: - log().warn("{}", message); - break; - case LogLevel::ERROR: - log().error("{}", message); - break; - } -} - -class PeriodicTimer : public google::api_manager::PeriodicTimer, - public Logger::Loggable { - private: - Server::Instance &server_; - Event::TimerPtr timer_; - - public: - PeriodicTimer(Server::Instance &server) : server_(server) {} - ~PeriodicTimer() { Stop(); } - void Stop() { - if (timer_) { - timer_->disableTimer(); - timer_ = nullptr; - } - } - void Schedule(std::chrono::milliseconds interval, - std::function continuation) { - Stop(); - timer_ = server_.dispatcher().createTimer([this, continuation, interval]() { - continuation(); - Schedule(interval, continuation); - }); - timer_->enableTimer(interval); - } -}; - -std::unique_ptr Env::StartPeriodicTimer( - std::chrono::milliseconds interval, std::function continuation) { - log().debug("start periodic timer"); - auto single = new PeriodicTimer(server); - single->Schedule(interval, continuation); - std::unique_ptr timer(single); - return timer; -} - -static const LowerCaseString kApiManagerUrl("x-api-manager-url"); -static const LowerCaseString kGrpcTEKey("te"); -static const std::string kGrpcTEValue("trailers"); - -class HTTPRequest : public Http::Message { - private: - HeaderMapImpl header_map_; - Buffer::OwnedImpl body_; - - public: - HTTPRequest(google::api_manager::HTTPRequest *request) - : body_(request->body()) { - header_map_.addStaticKey(Headers::get().Method, request->method()); - - size_t path_pos = request->url().find('/', 8); - if (path_pos == std::string::npos) { - header_map_.addStaticKey(Headers::get().Path, "/"); - } else { - header_map_.addStaticKey(Headers::get().Path, - request->url().substr(path_pos)); - } - - header_map_.addStaticKey(Headers::get().Scheme, "http"); - header_map_.addStaticKey(Headers::get().Host, "localhost"); - header_map_.addStaticKey(Headers::get().ContentLength, body_.length()); - header_map_.addStaticKey(kApiManagerUrl, request->url()); - for (const auto header : request->request_headers()) { - LowerCaseString lower_key(header.first); - HeaderString key, value; - key.setCopy(lower_key.get().data(), lower_key.get().size()); - value.setCopy(header.second.data(), header.second.size()); - header_map_.addViaMove(std::move(key), std::move(value)); - } - } - virtual HeaderMap &headers() override { return header_map_; } - virtual Buffer::Instance *body() override { return &body_; } - virtual void body(Buffer::InstancePtr &&body) override {} - virtual HeaderMap *trailers() override { return nullptr; } - virtual void trailers(HeaderMapPtr &&trailers) override {} - virtual std::string bodyAsString() const override { return ""; } -}; - -class HTTPRequestCallbacks : public AsyncClient::Callbacks { - private: - std::unique_ptr request_; - std::unique_ptr sent_request_; - - public: - HTTPRequestCallbacks( - std::unique_ptr &&request) - : request_(std::move(request)) {} - virtual void onSuccess(MessagePtr &&response) override { - google::api_manager::utils::Status status( - std::stoi(response->headers().Status()->value().c_str()), ""); - std::map headers; - response->headers().iterate( - [&](const HeaderEntry &header, void *) -> void { - // TODO: fix it - // headers.emplace(header.key().c_str(), header.value().c_str()); - }, - nullptr); - request_->OnComplete(status, std::move(headers), response->bodyAsString()); - delete this; - } - virtual void onFailure(AsyncClient::FailureReason reason) override { - google::api_manager::utils::Status status(-1, - "Cannot connect to HTTP server."); - std::map headers; - request_->OnComplete(status, std::move(headers), ""); - delete this; - } -}; - -namespace { -// Copy the code here from envoy/grpc/common.cc -Buffer::InstancePtr SerializeGrpcBody(const std::string &body_str) { - // http://www.grpc.io/docs/guides/wire.html - Buffer::InstancePtr body(new Buffer::OwnedImpl()); - uint8_t compressed = 0; - body->add(&compressed, sizeof(compressed)); - uint32_t size = htonl(body_str.size()); - body->add(&size, sizeof(size)); - body->add(body_str); - return body; -} -Http::MessagePtr PrepareGrpcHeaders(const std::string &upstream_cluster, - const std::string &service_full_name, - const std::string &method_name) { - Http::MessagePtr message(new Http::RequestMessageImpl()); - message->headers().insertMethod().value( - Http::Headers::get().MethodValues.Post); - message->headers().insertPath().value( - fmt::format("/{}/{}", service_full_name, method_name)); - message->headers().insertHost().value(upstream_cluster); - message->headers().insertContentType().value(Grpc::Common::GRPC_CONTENT_TYPE); - message->headers().addStatic(kGrpcTEKey, kGrpcTEValue); - return message; -} -} // annoymous namespace - -class GrpcRequestCallbacks : public AsyncClient::Callbacks { - private: - Env *env_; - std::unique_ptr request_; - - public: - GrpcRequestCallbacks( - Env *env, std::unique_ptr &&request) - : env_(env), request_(std::move(request)) {} - virtual void onSuccess(MessagePtr &&response) override { - google::api_manager::utils::Status status( - std::stoi(response->headers().Status()->value().c_str()), ""); - Grpc::Common::validateResponse(*response); - env_->LogInfo("pass validate"); - // remove 5 bytes of grpc header - response->body()->drain(5); - request_->OnComplete(status, response->bodyAsString()); - delete this; - } - virtual void onFailure(AsyncClient::FailureReason reason) override { - google::api_manager::utils::Status status(-1, - "Cannot connect to gRPC server."); - request_->OnComplete(status, ""); - delete this; - } -}; - -void Env::RunHTTPRequest( - std::unique_ptr request) { - auto &client = cm_.httpAsyncClientForCluster("api_manager"); - - MessagePtr message{new HTTPRequest(request.get())}; - HTTPRequestCallbacks *callbacks = - new HTTPRequestCallbacks(std::move(request)); - client.send( - std::move(message), *callbacks, - Optional(std::chrono::milliseconds(10000))); -} - -void Env::RunGRPCRequest( - std::unique_ptr request) { - auto &client = cm_.httpAsyncClientForCluster(request->server()); - - Http::MessagePtr message = - PrepareGrpcHeaders("localhost", request->service(), request->method()); - message->body(SerializeGrpcBody(request->body())); - auto callbacks = new GrpcRequestCallbacks(this, std::move(request)); - client.send( - std::move(message), *callbacks, - Optional(std::chrono::milliseconds(10000))); -} -} -} diff --git a/src/envoy/prototype/api_manager_env.h b/src/envoy/prototype/api_manager_env.h deleted file mode 100644 index 0ae8136f5be..00000000000 --- a/src/envoy/prototype/api_manager_env.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "precompiled/precompiled.h" - -#include "common/common/logger.h" -#include "contrib/endpoints/include/api_manager/env_interface.h" -#include "envoy/upstream/cluster_manager.h" -#include "server/server.h" - -namespace Http { -namespace ApiManager { - -class Env : public google::api_manager::ApiManagerEnvInterface, - public Logger::Loggable { - private: - Server::Instance& server; - Upstream::ClusterManager& cm_; - - public: - Env(Server::Instance& server) - : server(server), cm_(server.clusterManager()){}; - - virtual void Log(LogLevel level, const char* message) override; - virtual std::unique_ptr - StartPeriodicTimer(std::chrono::milliseconds interval, - std::function continuation) override; - virtual void RunHTTPRequest( - std::unique_ptr request) override; - virtual void RunGRPCRequest( - std::unique_ptr request) override; -}; -} -} diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc deleted file mode 100644 index 3983a4d8b41..00000000000 --- a/src/envoy/prototype/api_manager_filter.cc +++ /dev/null @@ -1,346 +0,0 @@ -#include "precompiled/precompiled.h" - -#include "api_manager_env.h" -#include "common/common/logger.h" -#include "common/grpc/common.h" -#include "common/http/filter/ratelimit.h" -#include "common/http/headers.h" -#include "common/http/utility.h" -#include "contrib/endpoints/include/api_manager/api_manager.h" -#include "contrib/endpoints/src/grpc/transcoding/transcoder.h" -#include "envoy/server/instance.h" -#include "server/config/network/http_connection_manager.h" - -namespace Http { -namespace ApiManager { -namespace { - -// Define lower case string for X-Forwarded-Host. -const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); - -} // namespace - -std::string ReadFile(const std::string& file_name) { - std::ifstream t(file_name); - std::string content((std::istreambuf_iterator(t)), - std::istreambuf_iterator()); - return content; -} - -class Config : public Logger::Loggable { - private: - google::api_manager::ApiManagerFactory api_manager_factory_; - std::shared_ptr api_manager_; - Upstream::ClusterManager& cm_; - - public: - Config(const Json::Object& config, Server::Instance& server) - : cm_(server.clusterManager()) { - std::string service_config_content; - if (config.hasObject("service_config")) { - const std::string service_config = config.getString("service_config"); - service_config_content = ReadFile(service_config); - } else { - log().error( - "Service_config is required but not specified in the config: {}", - __func__); - } - - std::string server_config_content; - if (config.hasObject("server_config")) { - const std::string server_config = config.getString("server_config"); - server_config_content = ReadFile(server_config); - } - std::unique_ptr env( - new Env(server)); - - api_manager_ = api_manager_factory_.GetOrCreateApiManager( - std::move(env), service_config_content, server_config_content); - - api_manager_->Init(); - log().debug("Called ApiManager::Config constructor: {}", __func__); - } - - std::shared_ptr& api_manager() { - return api_manager_; - } -}; - -typedef std::shared_ptr ConfigPtr; - -class Request : public google::api_manager::Request { - private: - HeaderMap& header_map_; - std::string downstream_address_; - std::string virtual_host_; - bool query_parsed_; - std::map query_params_; - - public: - Request(HeaderMap& header_map, const std::string& downstream_address, - const std::string& virtual_host) - : header_map_(header_map), - downstream_address_(downstream_address), - virtual_host_(virtual_host), - query_parsed_(false) {} - virtual std::string GetRequestHTTPMethod() override { - return header_map_.Method()->value().c_str(); - } - virtual std::string GetRequestPath() override { - return header_map_.Path()->value().c_str(); - } - virtual std::string GetUnparsedRequestPath() override { - return header_map_.Path()->value().c_str(); - } - - virtual std::string GetClientIP() override { - if (!header_map_.ForwardedFor()) { - return downstream_address_; - } - std::vector xff_address_list = - StringUtil::split(header_map_.ForwardedFor()->value().c_str(), ','); - if (xff_address_list.empty()) { - return downstream_address_; - } - return xff_address_list.front(); - } - - virtual std::string GetClientHost() override { - const HeaderEntry* entry = header_map_.get(kHeaderNameXFH); - if (entry == nullptr) { - return virtual_host_; - } - auto xff_list = StringUtil::split(entry->value().c_str(), ','); - if (xff_list.empty()) { - return virtual_host_; - } - return xff_list.back(); - } - - virtual bool FindQuery(const std::string& name, std::string* query) override { - if (!query_parsed_) { - auto header = header_map_.Path(); - if (header != nullptr) { - std::string path = header->value().c_str(); - Utility::parseQueryString(path).swap(query_params_); - } - query_parsed_ = true; - } - auto entry = query_params_.find(name); - if (entry == query_params_.end()) { - return false; - } - *query = entry->second; - return true; - } - - virtual bool FindHeader(const std::string& name, - std::string* header) override { - LowerCaseString key(name); - const HeaderEntry* entry = header_map_.get(key); - if (entry == nullptr) { - return false; - } - *header = entry->value().c_str(); - return true; - } - - virtual google::api_manager::protocol::Protocol GetRequestProtocol() - override { - return google::api_manager::protocol::Protocol::HTTP; - } - virtual google::api_manager::utils::Status AddHeaderToBackend( - const std::string& key, const std::string& value) override { - return google::api_manager::utils::Status::OK; - } - virtual void SetAuthToken(const std::string& auth_token) override {} - - virtual int64_t GetGrpcRequestBytes() { return 0; } - virtual int64_t GetGrpcResponseBytes() { return 0; } - virtual int64_t GetGrpcRequestMessageCounts() { return 0; } - virtual int64_t GetGrpcResponseMessageCounts() { return 0; } - virtual std::string GetQueryParameters() { return ""; } -}; - -class Response : public google::api_manager::Response { - const AccessLog::RequestInfo& request_info_; - - public: - Response(const AccessLog::RequestInfo& request_info) - : request_info_(request_info) {} - - google::api_manager::utils::Status GetResponseStatus() { - return google::api_manager::utils::Status::OK; - } - - std::size_t GetRequestSize() { return request_info_.bytesReceived(); } - - std::size_t GetResponseSize() { return request_info_.bytesSent(); } - - google::api_manager::utils::Status GetLatencyInfo( - google::api_manager::service_control::LatencyInfo* info) { - info->request_time_ms = request_info_.duration().count(); - return google::api_manager::utils::Status::OK; - } -}; - -const Http::HeaderMapImpl BadRequest{{Http::Headers::get().Status, "400"}}; - -class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { - private: - std::shared_ptr api_manager_; - std::unique_ptr - request_handler_; - - enum State { NotStarted, Calling, Complete, Responded }; - State state_; - - StreamDecoderFilterCallbacks* decoder_callbacks_; - StreamEncoderFilterCallbacks* encoder_callbacks_; - - bool initiating_call_; - - std::string getRouteVirtualHost(HeaderMap& headers) const { - const Router::Route* route = decoder_callbacks_->route(); - if (route && route->routeEntry()) { - return route->routeEntry()->virtualHost().name(); - } - return ""; - } - - public: - Instance(ConfigPtr config) - : api_manager_(config->api_manager()), - state_(NotStarted), - initiating_call_(false) { - Log().debug("Called ApiManager::Instance : {}", __func__); - } - - FilterHeadersStatus decodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr request( - new Request(headers, decoder_callbacks_->downstreamAddress(), - getRouteVirtualHost(headers))); - request_handler_ = api_manager_->CreateRequestHandler(std::move(request)); - state_ = Calling; - initiating_call_ = true; - request_handler_->Check([this](google::api_manager::utils::Status status) { - completeCheck(status); - }); - initiating_call_ = false; - - if (state_ == Complete) { - return FilterHeadersStatus::Continue; - } - Log().debug("Called ApiManager::Instance : {} Stop", __func__); - return FilterHeadersStatus::StopIteration; - } - - FilterDataStatus decodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, - data.length(), end_stream); - if (state_ == Calling) { - return FilterDataStatus::StopIterationAndBuffer; - } - return FilterDataStatus::Continue; - } - - FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - if (state_ == Calling) { - return FilterTrailersStatus::StopIteration; - } - return FilterTrailersStatus::Continue; - } - void setDecoderFilterCallbacks( - StreamDecoderFilterCallbacks& callbacks) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - decoder_callbacks_ = &callbacks; - decoder_callbacks_->addResetStreamCallback( - [this]() { state_ = Responded; }); - } - void completeCheck(const google::api_manager::utils::Status& status) { - Log().debug("Called ApiManager::Instance : check complete {}", - status.ToJson()); - if (!status.ok() && state_ != Responded) { - state_ = Responded; - decoder_callbacks_->dispatcher().post([this, status]() { - Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), - status.ToJson()); - }); - return; - } - state_ = Complete; - if (!initiating_call_) { - decoder_callbacks_->dispatcher().post( - [this]() { decoder_callbacks_->continueDecoding(); }); - } - } - - virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterHeadersStatus::Continue; - } - virtual FilterDataStatus encodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterDataStatus::Continue; - } - virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterTrailersStatus::Continue; - } - virtual void setEncoderFilterCallbacks( - StreamEncoderFilterCallbacks& callbacks) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - encoder_callbacks_ = &callbacks; - } - - virtual void log(const HeaderMap* request_headers, - const HeaderMap* response_headers, - const AccessLog::RequestInfo& request_info) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr response( - new Response(request_info)); - request_handler_->Report(std::move(response), []() {}); - } - - spdlog::logger& Log() { - static spdlog::logger& instance = - Logger::Registry::getLog(Logger::Id::http); - return instance; - } -}; -} -} - -namespace Server { -namespace Configuration { - -class ApiManagerConfig : public HttpFilterConfigFactory { - public: - HttpFilterFactoryCb tryCreateFilterFactory( - HttpFilterType type, const std::string& name, const Json::Object& config, - const std::string&, Server::Instance& server) override { - if (type != HttpFilterType::Both || name != "esp") { - return nullptr; - } - - Http::ApiManager::ConfigPtr api_manager_config( - new Http::ApiManager::Config(config, server)); - return [api_manager_config]( - Http::FilterChainFactoryCallbacks& callbacks) -> void { - std::shared_ptr instance( - new Http::ApiManager::Instance(api_manager_config)); - callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); - callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); - }; - } -}; - -static RegisterHttpFilterConfigFactory register_; -} -} diff --git a/src/envoy/prototype/dummy_api_manager_cluster.py b/src/envoy/prototype/dummy_api_manager_cluster.py deleted file mode 100644 index 3f83476aae8..00000000000 --- a/src/envoy/prototype/dummy_api_manager_cluster.py +++ /dev/null @@ -1,91 +0,0 @@ -from BaseHTTPServer import BaseHTTPRequestHandler -from BaseHTTPServer import HTTPServer -from subprocess import Popen, PIPE, STDOUT - -PUBLIC_KEY = '''{ - "keys": [ - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "686f442dead739e241342519849bd0d140707a86", - "n": "yVGRzBnVmCHC9uMH_mCna7FiSKOjMEsDdgf8hds41KYfUHH9cp4P41iQEBnqPYvQiMYaZDuatkMDh25ukmkSyfLwJIVQpnHgMDwoZ1cx83Z0PwC8wAAJYqKyItKNfFN4UJ6DZIOVU-Iqgxi8VOtGwMNx2LiD1NoFVfXyz52UJ_QLiUGzErVwTGv4UD6NtaWKFkctTnEG-9rZvDF8QknnzxAomVa2OcV8OHeszx6N8UE1tm9Kq4xj2Uu8D3dDrfu2jr45Pi6RHIZOTAnu8G7wTDNaiGbENrbHSk6WAjLZBOcWZj9SDlDlwH2xFoKdpNKRmLKpPQblHem_y1KRYwvJTw", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "d57d66bfbff089769e3545589572d3b53955a8cb", - "n": "zX99YCqicbn_j-5YTJ-2FsgONUg7cmqJiwvHXrUopRvn2Tukrd0B5Sg-Rq1hdZTYgIym7lSMw_zLxIQCH54sUfydX5MMWr6FOxVUbYl-E0Oko85Yer9dFv61rN0USj_A12QRMmjCZkcqH_6MWWuA1QWaejyStopjpLEYnUD3bP6oS604eZWkkOp8Nu-Vg4NqqX7ZClIcQqe03xv3sFHiPuhB-qaifhpIPpKCiYSKxAY4_GxwCJ_ml_uJ5k1tIrDykAlie6aWxv8hogOXrQmNRCO2Qcumwb7d9cci1UxsEYOtpZxhTiZqWsbrBxwfvLqU_rvsCT8vOjrBPOkTtYl22w", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "516ca9a83b130d4b605684fe7df83b43f33fba43", - "n": "rSLxCvtRLoJUt2xPTSjDgN8_7kuBLsZz0JD4HRiwvId_IYY-T5MsUirBx01AtI3KRlmeQ-1QIvET4cNN99QL8U6e7Hh75VWJvQhh0_TFdKbv_75b81-rzx-EvIoVe0czeY3QeRp3DDORRk9o7xgri6U3VfT2zcQIDAPY14h3wLHE5DQuH9EpbxjS_wNrQxRYZH_mWgmWU7h001WHWRDKdEIgtzQkinKOTig90Wy4pt_vgMGcv0mh_wYvIcmB62Qj1sHUUOa5NJWUaZLC4Stj7CH0dLiBrVqg3JFqvO60Oo3wrcdmNQl0RaRHFjBYLameUPSP6M2BevfCRx8-Ix26rw", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "33a681b36b8983913b05f8d5eab165c60694b6fc", - "n": "pRbWhLXIOvF1Naxh5n2evZqpJ3kPEFL4b-jHRHOhMnzqsi_aeQtIwYLVM8zdYYfgCxRq8umG1YwMenBKKPzRWr4MkFAB64O3UPgdyg-3je-fgCMziqS_3KH7AXHekxG_ZHpwbkgilRMtJNiDnSZWGad8XAfW3VFct4RqRAaf7h-6Za0IM7R3u4VYkUfosNqKtoDJDQrng9Nbv9ryUk8u1WikKF0M1r-wrZoDA7QFRAFkbdfYyvHL2daUflNDIXmFkUeHGGApMlJQ3wJk7Tln4txGMSUdJoD3JWEyKa2W1WshtqBHnQb8VlL77H-ch9zbn5pGZCoJ0MsgJr0vjoea8w", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "986f2c30b9843defdc8f8debe138d640d1d67d9d", - "n": "qfE-u8I60zP_Vg3UsP_X9Qv1idkrnsXKkcQcsGWUdqs4jM2VixGBJDMJWbP55aJA93Sl49_dzAv_lr5-ctc1L04ke6nHX19EEBDtFTjGKyEdI-X1C-6HXCQmpI1XvUA8DzTOIZd5KEXJgBA9tpn6qkMAHzXyMHOfg8nhW36She4QggFjfF_RDKOA2-jRXDIjinOmIaLhVq4hsC9hCshhsfreLPw3HH8UhRONbkoFU_8ZBgAQLdVB1TQwp-ZDiVyh9od9a-RmfIiUl27AK5LDrtpVRCtXj6bv9OMx4QWVX8G-NfQexDwAp1pICaE5qYyJbiIK25E07vPTJ1ZtMpodFQ", - "e": "AQAB" - } - ] -} -''' - -def print_body(binary, expected): - p = Popen(['bazel-bin/src/tools/service_control_json_gen', - '--text', - '--'+expected, - '--stdin'], stdin=PIPE) - p.communicate(input=binary) - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - self.handle_request("GET") - - def do_POST(self): - self.handle_request("POST") - - - - def handle_request(self, method): - print(method, self.path, self.headers.items()) - url = self.headers.get('x-api-manager-url', '') - if url == 'https://www.googleapis.com/service_accounts/v1/jwk/loadtest@esp-test-client.iam.gserviceaccount.com': - self.send_response(200) - self.send_header('Content-Length', str(len(PUBLIC_KEY))) - self.end_headers() - self.wfile.write(PUBLIC_KEY) - else: - #content_len = self.headers.getheader('content-length', 0) - #post_body = self.rfile.read(int(content_len)) - if url.endswith(":report"): - print_body(post_body, "report_request") - elif url.endswith(":check"): - print_body(post_body, "check_request") - - self.send_response(200) - self.send_header('Content-Length', "0") - self.end_headers() - - -if __name__ == '__main__': - server = HTTPServer(('localhost', 8081), Handler) - print 'Starting server, use to stop' - server.serve_forever() diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf deleted file mode 100644 index 27ea9f1b397..00000000000 --- a/src/envoy/prototype/envoy-esp.conf +++ /dev/null @@ -1,89 +0,0 @@ -{ - "listeners": [ - { - "port": 9092, - "bind_to_port": true, - "use_original_dst": true, - "filters": [] - }, - { - "port": 9090, - "bind_to_port": false, - "filters": [ - { - "type": "read", - "name": "http_connection_manager", - "config": { - "codec_type": "auto", - "stat_prefix": "ingress_http", - "route_config": { - "virtual_hosts": [ - { - "name": "backend", - "domains": ["*"], - "routes": [ - { - "timeout_ms": 0, - "prefix": "/", - "cluster": "service1" - } - ] - } - ] - }, - "access_log": [ - { - "path": "/dev/stdout" - } - ], - "filters": [ - { - "type": "both", - "name": "esp", - "config": { - "service_config": "src/envoy/prototype/generic_service_config.json", - "server_config": "src/envoy/prototype/server_config.pb.txt" - } - }, - { - "type": "decoder", - "name": "router", - "config": {} - } - ] - } - } - ] - } - ], - "admin": { - "access_log_path": "/dev/stdout", - "port": 9001 - }, - "cluster_manager": { - "clusters": [ - { - "name": "service1", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "hosts": [ - { - "url": "tcp://localhost:8080" - } - ] - }, - { - "name": "api_manager", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "hosts": [ - { - "url": "tcp://localhost:8081" - } - ] - } - ] - } -} diff --git a/src/envoy/prototype/generic_service_config.json b/src/envoy/prototype/generic_service_config.json deleted file mode 100644 index ac926e5e4c2..00000000000 --- a/src/envoy/prototype/generic_service_config.json +++ /dev/null @@ -1,736 +0,0 @@ -{ - "apis": [ - { - "methods": [ - { - "name": "Get", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Delete", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Patch", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Post", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Put", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "generic_qiwzhang_app_1_appspot_com_1_0_0", - "sourceContext": { - "fileName": "sss.yaml" - }, - "version": "1.0.0" - } - ], - "authentication": {}, - "backend": { - "rules": [ - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "configVersion": 3, - "context": {}, - "control": { - "environment": "servicecontrol.googleapis.com" - }, - "customError": {}, - "documentation": {}, - "enums": [ - { - "enumvalue": [ - { - "name": "NULL_VALUE" - } - ], - "name": "google.protobuf.NullValue", - "sourceContext": { - "fileName": "struct.proto" - } - } - ], - "http": { - "rules": [ - { - "get": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "delete": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "patch": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "post": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "put": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "id": "2017-01-06r0", - "logging": { - "producerDestinations": [ - { - "logs": [ - "endpoints_log" - ], - "monitoredResource": "api" - } - ] - }, - "logs": [ - { - "name": "endpoints_log" - } - ], - "metrics": [ - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/error_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/error_count", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/end_user" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/end_user_country" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user_country", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/referer" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_referer", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/consumer_id" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_request_count_by_consumer", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/quota_used_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/quota_billable_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/consumer_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_request_sizes_by_consumer", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/consumer_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_response_sizes_by_consumer", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/error_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/quota_used_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_request_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_response_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_request_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_response_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_durations", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_durations", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/response_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/response_bytes", - "valueType": "INT64" - } - ], - "monitoredResources": [ - { - "labels": [ - { - "key": "cloud.googleapis.com/location" - }, - { - "key": "cloud.googleapis.com/uid" - }, - { - "key": "serviceruntime.googleapis.com/api_version" - }, - { - "key": "serviceruntime.googleapis.com/api_method" - }, - { - "key": "serviceruntime.googleapis.com/consumer_project" - }, - { - "key": "cloud.googleapis.com/project" - }, - { - "key": "cloud.googleapis.com/service" - } - ], - "type": "api" - } - ], - "monitoring": { - "consumerDestinations": [ - { - "metrics": [ - "serviceruntime.googleapis.com/api/consumer/request_count", - "serviceruntime.googleapis.com/api/consumer/error_count", - "serviceruntime.googleapis.com/api/consumer/quota_used_count", - "serviceruntime.googleapis.com/api/consumer/total_latencies", - "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/consumer/backend_latencies", - "serviceruntime.googleapis.com/api/consumer/request_sizes", - "serviceruntime.googleapis.com/api/consumer/response_sizes", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user_country", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_referer", - "serviceruntime.googleapis.com/api/consumer/streaming_request_message_counts", - "serviceruntime.googleapis.com/api/consumer/streaming_response_message_counts", - "serviceruntime.googleapis.com/api/consumer/streaming_durations", - "serviceruntime.googleapis.com/api/consumer/request_bytes", - "serviceruntime.googleapis.com/api/consumer/response_bytes" - ], - "monitoredResource": "api" - } - ], - "producerDestinations": [ - { - "metrics": [ - "serviceruntime.googleapis.com/api/producer/request_count", - "serviceruntime.googleapis.com/api/producer/error_count", - "serviceruntime.googleapis.com/api/producer/total_latencies", - "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/producer/backend_latencies", - "serviceruntime.googleapis.com/api/producer/request_sizes", - "serviceruntime.googleapis.com/api/producer/response_sizes", - "serviceruntime.googleapis.com/api/producer/top_request_count_by_consumer", - "serviceruntime.googleapis.com/api/producer/top_request_sizes_by_consumer", - "serviceruntime.googleapis.com/api/producer/top_response_sizes_by_consumer", - "serviceruntime.googleapis.com/api/producer/streaming_request_message_counts", - "serviceruntime.googleapis.com/api/producer/streaming_response_message_counts", - "serviceruntime.googleapis.com/api/producer/streaming_durations", - "serviceruntime.googleapis.com/api/producer/request_bytes", - "serviceruntime.googleapis.com/api/producer/response_bytes", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/error_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/total_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/quota_used_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/backend_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_sizes", - "serviceruntime.googleapis.com/api/producer/by_consumer/response_sizes" - ], - "monitoredResource": "api" - } - ] - }, - "name": "generic.qiwzhang-app-1.appspot.com", - "producerProjectId": "qiwzhang-app-1", - "systemParameters": {}, - "title": "Basic HTTP service configuration", - "types": [ - { - "fields": [ - { - "cardinality": "CARDINALITY_REPEATED", - "jsonName": "values", - "kind": "TYPE_MESSAGE", - "name": "values", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "google.protobuf.ListValue", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_REPEATED", - "jsonName": "fields", - "kind": "TYPE_MESSAGE", - "name": "fields", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.Struct.FieldsEntry" - } - ], - "name": "google.protobuf.Struct", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "key", - "kind": "TYPE_STRING", - "name": "key", - "number": 1 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "value", - "kind": "TYPE_MESSAGE", - "name": "value", - "number": 2, - "typeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "google.protobuf.Struct.FieldsEntry", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "name": "google.protobuf.Empty", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "nullValue", - "kind": "TYPE_ENUM", - "name": "null_value", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.NullValue" - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "numberValue", - "kind": "TYPE_DOUBLE", - "name": "number_value", - "number": 2 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "stringValue", - "kind": "TYPE_STRING", - "name": "string_value", - "number": 3 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "boolValue", - "kind": "TYPE_BOOL", - "name": "bool_value", - "number": 4 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "structValue", - "kind": "TYPE_MESSAGE", - "name": "struct_value", - "number": 5, - "typeUrl": "type.googleapis.com/google.protobuf.Struct" - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "listValue", - "kind": "TYPE_MESSAGE", - "name": "list_value", - "number": 6, - "typeUrl": "type.googleapis.com/google.protobuf.ListValue" - } - ], - "name": "google.protobuf.Value", - "sourceContext": { - "fileName": "struct.proto" - } - } - ], - "usage": { - "rules": [ - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "visibility": {} -} diff --git a/src/envoy/prototype/server_config.pb.txt b/src/envoy/prototype/server_config.pb.txt deleted file mode 100644 index 19cb6cbb2c0..00000000000 --- a/src/envoy/prototype/server_config.pb.txt +++ /dev/null @@ -1,6 +0,0 @@ -cloud_tracing_config { - force_disable: true -} -mixer_options { - mixer_server: "localhost:9091" -} diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 94f371ebc7d..ca7e9468ded 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -431,8 +431,8 @@ cc_library( native.new_http_archive( name = "nghttp2_tar", - url = "https://github.com/nghttp2/nghttp2/releases/download/v1.14.1/nghttp2-1.14.1.tar.gz", - strip_prefix = "nghttp2-1.14.1", + url = "https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz", + strip_prefix = "nghttp2-1.20.0", build_file_content = BUILD, ) @@ -442,6 +442,127 @@ cc_library( actual = "@nghttp2_tar//:nghttp2", ) +def ares_repositories(bind=True): + BUILD = """ +cc_library( + name = "ares", + srcs = [ + "ares__close_sockets.c", + "ares__get_hostent.c", + "ares__read_line.c", + "ares__timeval.c", + "ares_cancel.c", + "ares_create_query.c", + "ares_data.c", + "ares_destroy.c", + "ares_expand_name.c", + "ares_expand_string.c", + "ares_fds.c", + "ares_free_hostent.c", + "ares_free_string.c", + "ares_getenv.c", + "ares_gethostbyaddr.c", + "ares_gethostbyname.c", + "ares_getnameinfo.c", + "ares_getopt.c", + "ares_getsock.c", + "ares_init.c", + "ares_library_init.c", + "ares_llist.c", + "ares_mkquery.c", + "ares_nowarn.c", + "ares_options.c", + "ares_parse_a_reply.c", + "ares_parse_aaaa_reply.c", + "ares_parse_mx_reply.c", + "ares_parse_naptr_reply.c", + "ares_parse_ns_reply.c", + "ares_parse_ptr_reply.c", + "ares_parse_soa_reply.c", + "ares_parse_srv_reply.c", + "ares_parse_txt_reply.c", + "ares_platform.c", + "ares_process.c", + "ares_query.c", + "ares_search.c", + "ares_send.c", + "ares_strcasecmp.c", + "ares_strdup.c", + "ares_strerror.c", + "ares_timeout.c", + "ares_version.c", + "ares_writev.c", + "bitncmp.c", + "inet_net_pton.c", + "inet_ntop.c", + "windows_port.c", + ], + hdrs = [ + "ares_config.h", + "ares.h", + "ares_build.h", + "ares_data.h", + "ares_dns.h", + "ares_getenv.h", + "ares_getopt.h", + "ares_inet_net_pton.h", + "ares_iphlpapi.h", + "ares_ipv6.h", + "ares_library_init.h", + "ares_llist.h", + "ares_nowarn.h", + "ares_platform.h", + "ares_private.h", + "ares_rules.h", + "ares_setup.h", + "ares_strcasecmp.h", + "ares_strdup.h", + "ares_version.h", + "ares_writev.h", + "bitncmp.h", + "nameser.h", + "setup_once.h", + ], + copts = [ + "-DHAVE_CONFIG_H", + ], + includes = ["."], + visibility = ["//visibility:public"], +) + +genrule( + name = "config", + srcs = glob(["**/*"]), + outs = ["ares_config.h"], + cmd = "pushd external/cares_git ; ./buildconf ; ./configure ; cp ares_config.h ../../$@", + visibility = ["//visibility:public"], +) + +genrule( + name = "ares_build", + srcs = [ + "ares_build.h.dist", + ], + outs = [ + "ares_build.h", + ], + cmd = "cp $(SRCS) $@", + visibility = ["//visibility:public"], +) +""" + + native.new_git_repository( + name = "cares_git", + remote = "https://github.com/c-ares/c-ares.git", + commit = "7691f773af79bf75a62d1863fd0f13ebf9dc51b1", # v1.12.0 + build_file_content = BUILD, + ) + + if bind: + native.bind( + name = "ares", + actual = "@cares_git//:ares", + ) def envoy_repositories(bind=True): libevent_repositories(bind) @@ -451,6 +572,7 @@ def envoy_repositories(bind=True): http_parser_repositories(bind) rapidjson_repositories(bind) nghttp2_repositories(bind) + ares_repositories(bind) BUILD = """ load("@protobuf_git//:protobuf.bzl", "cc_proto_library") @@ -541,6 +663,7 @@ cc_library( alwayslink=1, deps = [ ":envoy-ratelimit-pb", + "//external:ares", "//external:libssl", "//external:nghttp2", "//external:spdlog", @@ -629,6 +752,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "02c6fc97b4c21d25ab596a25208fbe283e927f6a", + commit = "70e5d651b55d356770529e5bee9c6b2707d9cf21", # 3/1/2017 build_file_content = BUILD, ) From cfdb30323aa72b680bab1664035a1bbc200379c0 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Mon, 13 Mar 2017 10:34:11 -0700 Subject: [PATCH 10/15] Merge from master to firebase (#159) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect (#38) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect * Fixed style. * Rebase Envoy (#41) * Update prototype to use iptables (#42) * Rebase to fixed Envoy (#43) * Handle HEAD request. (#34) * Handle HEAD request. * Try with GET if HEAD fails. * Address comments. * Format file. * Expose bazel target (#48) * Try again (#49) * Integrate with mixer client. (#55) * Integrate with mixer client. * Restore repositories.bzl back. * Add originIp and originHost attributes. (#56) * Add uuid-dev dependency in README.md (#45) * Extract originIp and OriginHost. (#57) * Extract originIp and OriginHost. * Make header x-forwarded-host const. * Update buckets for UI. (#58) * Update buckets for UI. * Only update time_distribution. * Add targetService attribute. (#59) * Use envoy new access_log handler for sending Report. (#60) * use access_log handler. * Not to use Loggable base class. * Update to the latest envoy with #396. (#61) * Fix tclap dependency fetching error (#62) * Integrate mixer client directly with envoy. (#66) * Integrate mixer client directly with envoy. * Send response header in Report. * rename filter name from esp to mixer. * add README. * Add release binary script. (#68) * Push tar.gz to GCS (#69) * Push tar.gz to GCS * Rename envoy_esp * Remove mixer_client from api_manager. (#72) * Update mixer client SHA. (#74) * Update readme. (#73) * Adds Jenkinsfile and updates release-binary to create a SHA. (#71) * Adds Jenkinsfile and update release-binary * Update Jenkinsfile and gitignore * Fixes typo and use normal build Node * Uses default bazel config * Using batch mode * Update bazel memory settings * Do not use Jenkins bazel env * Set .bazelrc for postsubmit * Update grpc and protobuf (#70) * protobuf v3.2.0 * grpc v1.1.1 * Align auth lib with grpc 1.1.1 * Add sourceService. (#78) * Add script to build docker image. (#77) * Add script to build docker image. * Add start_envoy for docker image. * Use official attribute names (#80) * Use official attribute names * fix format * Creates a KEY for mixer client dep. Updates release-binary (#79) * Updated mixer repo to use a key for commit * release-binary skip build if file exists. * Update src/envoy/mixer/README. (#82) * Fix src/envoy/mixer/README.md (#85) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Not to use api_key if its service is not actived. (#109) * Update envoy and add c-ares (#107) * Update envoy and add c-ares depedencies * Update release script with debug and normal binary * remove debug ls * formatting * Send StatusCode Attributes to Mixer. (#110) * Add send_attribute filter. (#115) * Add send_attribute filter. * Fix format * rename variable serialized_attributes_ * Address the comments. * Fail request if api_key is not valid. (#116) * Fail request if api_key is not valid. * Format code. * Update comments. * Address comment. * Rename response.http.code (#125) * Send headers as string map. (#129) * Send headers as string map. * Remove origin.ip and origin.host. * Fix format * unify bazel's docker build targets with other istio repos (#127) * update base debug docker image reference (#133) * Update postsubmit to create docker images (#132) * Adding config release for bazel build (#135) * Fix mixer client crash. (#136) * Get mixerclient with response parsing. (#138) * Update nghttp2 to sync with envoy (#140) * Fix src/envoy/mixer/README.md * Update nghttp2 to sync with envoy * update * fix typo * Populate origin.user attribute from the SAN field of client cert (#142) * Test * test * test * revert file * address comments * test * fix typo * fix format * fix format * Update to latest mixer_client. (#145) * Update to latest mixer_client. * Updated the sha. * Not call report if decodeHeaders is not called. (#150) * Update mixerclient with sync-ed grpc write and fail-fast. (#155) * Update mixerclient with sync-ed write and fail-fast. * Update to latest test. * Update again * Update envoy to PR553 (#156) * Update envoy to PR553 * Update libevent to 2.1.8 * Update the Commit id for envoy --- src/envoy/mixer/http_control.cc | 74 ++++++++++---------------------- src/envoy/mixer/http_control.h | 2 +- src/envoy/mixer/http_filter.cc | 14 +++++- src/envoy/mixer/repositories.bzl | 2 +- src/envoy/repositories.bzl | 9 ++-- 5 files changed, 41 insertions(+), 60 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 6e64659c2bb..3b97d20a35f 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -31,59 +31,24 @@ namespace Mixer { namespace { // Define attribute names -const std::string kRequestPath = "request.path"; +const std::string kOriginUser = "origin.user"; + +const std::string kRequestHeaders = "request.headers"; const std::string kRequestHost = "request.host"; +const std::string kRequestPath = "request.path"; const std::string kRequestSize = "request.size"; const std::string kRequestTime = "request.time"; -const std::string kRequestHeaders = "request.headers"; const std::string kResponseHeaders = "response.headers"; +const std::string kResponseHttpCode = "response.http.code"; +const std::string kResponseLatency = "response.latency"; const std::string kResponseSize = "response.size"; const std::string kResponseTime = "response.time"; -const std::string kResponseLatency = "response.latency"; -const std::string kResponseHttpCode = "response.http.code"; - -Attributes::Value StringValue(const std::string& str) { - Attributes::Value v; - v.type = Attributes::Value::STRING; - v.str_v = str; - return v; -} - -Attributes::Value StringMapValue( - std::map&& string_map) { - Attributes::Value v; - v.type = Attributes::Value::STRING_MAP; - v.string_map_v.swap(string_map); - return v; -} - -Attributes::Value Int64Value(int64_t value) { - Attributes::Value v; - v.type = Attributes::Value::INT64; - v.value.int64_v = value; - return v; -} - -Attributes::Value TimeValue( - std::chrono::time_point value) { - Attributes::Value v; - v.type = Attributes::Value::TIME; - v.time_v = value; - return v; -} - -Attributes::Value DurationValue(std::chrono::nanoseconds value) { - Attributes::Value v; - v.type = Attributes::Value::DURATION; - v.duration_nanos_v = value; - return v; -} void SetStringAttribute(const std::string& name, const std::string& value, Attributes* attr) { if (!value.empty()) { - attr->attributes[name] = StringValue(value); + attr->attributes[name] = Attributes::StringValue(value); } } @@ -103,39 +68,43 @@ void FillRequestHeaderAttributes(const HeaderMap& header_map, Attributes* attr) { SetStringAttribute(kRequestPath, header_map.Path()->value().c_str(), attr); SetStringAttribute(kRequestHost, header_map.Host()->value().c_str(), attr); - attr->attributes[kRequestTime] = TimeValue(std::chrono::system_clock::now()); + attr->attributes[kRequestTime] = + Attributes::TimeValue(std::chrono::system_clock::now()); attr->attributes[kRequestHeaders] = - StringMapValue(ExtractHeaders(header_map)); + Attributes::StringMapValue(ExtractHeaders(header_map)); } void FillResponseHeaderAttributes(const HeaderMap* header_map, Attributes* attr) { if (header_map) { attr->attributes[kResponseHeaders] = - StringMapValue(ExtractHeaders(*header_map)); + Attributes::StringMapValue(ExtractHeaders(*header_map)); } - attr->attributes[kResponseTime] = TimeValue(std::chrono::system_clock::now()); + attr->attributes[kResponseTime] = + Attributes::TimeValue(std::chrono::system_clock::now()); } void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, int check_status_code, Attributes* attr) { if (info.bytesReceived() >= 0) { - attr->attributes[kRequestSize] = Int64Value(info.bytesReceived()); + attr->attributes[kRequestSize] = + Attributes::Int64Value(info.bytesReceived()); } if (info.bytesSent() >= 0) { - attr->attributes[kResponseSize] = Int64Value(info.bytesSent()); + attr->attributes[kResponseSize] = Attributes::Int64Value(info.bytesSent()); } if (info.duration().count() > 0) { - attr->attributes[kResponseLatency] = DurationValue( + attr->attributes[kResponseLatency] = Attributes::DurationValue( std::chrono::duration_cast(info.duration())); } if (info.responseCode().valid()) { attr->attributes[kResponseHttpCode] = - Int64Value(info.responseCode().value()); + Attributes::Int64Value(info.responseCode().value()); } else { - attr->attributes[kResponseHttpCode] = Int64Value(check_status_code); + attr->attributes[kResponseHttpCode] = + Attributes::Int64Value(check_status_code); } } @@ -170,8 +139,9 @@ void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { } void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, - DoneFunc on_done) { + std::string origin_user, DoneFunc on_done) { FillCheckAttributes(headers, &request_data->attributes); + SetStringAttribute(kOriginUser, origin_user, &request_data->attributes); log().debug("Send Check: {}", request_data->attributes.DebugString()); mixer_client_->Check(request_data->attributes, on_done); } diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index c5938cf369d..e9ddc734f45 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -42,7 +42,7 @@ class HttpControl final : public Logger::Loggable { // Make mixer check call. void Check(HttpRequestDataPtr request_data, HeaderMap& headers, - ::istio::mixer_client::DoneFunc on_done); + std::string origin_user, ::istio::mixer_client::DoneFunc on_done); // Make mixer report call. void Report(HttpRequestDataPtr request_data, diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index a990b0fd7e3..c99329aea4f 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -19,6 +19,7 @@ #include "common/http/headers.h" #include "common/http/utility.h" #include "envoy/server/instance.h" +#include "envoy/ssl/connection.h" #include "server/config/network/http_connection_manager.h" #include "src/envoy/mixer/http_control.h" #include "src/envoy/mixer/utils.h" @@ -151,8 +152,16 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { state_ = Calling; initiating_call_ = true; request_data_ = std::make_shared(); + + std::string origin_user; + Ssl::Connection* ssl = + const_cast(decoder_callbacks_->ssl()); + if (ssl != nullptr) { + origin_user = ssl->uriSanPeerCertificate(); + } + http_control_->Check( - request_data_, headers, + request_data_, headers, origin_user, wrapper([this](const Status& status) { completeCheck(status); })); initiating_call_ = false; @@ -180,6 +189,7 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { } return FilterTrailersStatus::Continue; } + void setDecoderFilterCallbacks( StreamDecoderFilterCallbacks& callbacks) override { Log().debug("Called Mixer::Instance : {}", __func__); @@ -227,6 +237,8 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { const HeaderMap* response_headers, const AccessLog::RequestInfo& request_info) override { Log().debug("Called Mixer::Instance : {}", __func__); + // If decodeHaeders() is not called, not to call Mixer report. + if (!request_data_) return; // Make sure not to use any class members at the callback. // The class may be gone when it is called. // Log() is a static function so it is OK. diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 2f665d71c4b..473e2650882 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "5b5745f29ac5a8babe79ada573defaa83f3bb9e7" +MIXER_CLIENT = "92a305961bcea90ed349ffdbb0ea299c6f6bacad" def mixer_client_repositories(bind=True): native.git_repository( diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index ca7e9468ded..dffa32438a0 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -62,6 +62,7 @@ event_srcs = [ "evthread.c", "evutil.c", "evutil_rand.c", + "evutil_time.c", "http.c", "listener.c", "log.c", @@ -86,9 +87,7 @@ cc_library( "defer-internal.h", "evbuffer-internal.h", "event-internal.h", - "event.h", "evthread-internal.h", - "evutil.h", "http-internal.h", "iocp-internal.h", "ipv6-internal.h", @@ -134,8 +133,8 @@ cc_library( native.new_http_archive( name = "libevent_git", - url = "https://github.com/libevent/libevent/releases/download/release-2.0.22-stable/libevent-2.0.22-stable.tar.gz", - strip_prefix = "libevent-2.0.22-stable", + url = "https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz", + strip_prefix = "libevent-2.1.8-stable", build_file_content = BUILD, ) @@ -752,6 +751,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "70e5d651b55d356770529e5bee9c6b2707d9cf21", # 3/1/2017 + commit = "9dcac8ca111ecc8da059d1f8d42eb766b44bacd6", # https://github.com/lyft/envoy/pull/553 build_file_content = BUILD, ) From e00681ab98ddb3bde4cd3a530fb14a47a24ba65c Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Wed, 29 Mar 2017 15:23:21 -0700 Subject: [PATCH 11/15] Allow for HTTP based function from Firebase rules (#202) * Allow for HTTP based function from Firebase rules * Fix code style check * Added more comments. * Fix style issues. * Address code review comments from Limin and Lizan. * Add more comments and address CR comments. * Fix a typo. * Address Wayne's CR comments. --- contrib/endpoints/src/api_manager/BUILD | 3 + .../api_manager/auth/service_account_token.h | 3 + .../src/api_manager/check_security_rules.cc | 217 ++-------- .../api_manager/check_security_rules_test.cc | 319 ++++++++++---- .../src/api_manager/context/request_context.h | 4 +- .../api_manager/context/service_context.cc | 12 + .../src/api_manager/firebase_rules/BUILD | 41 ++ .../firebase_rules/firebase_request.cc | 403 ++++++++++++++++++ .../firebase_rules/firebase_request.h | 197 +++++++++ .../api_manager/proto/security_rules.proto | 231 ++++++++-- .../src/api_manager/proto/server_config.proto | 1 + .../src/api_manager/utils/url_util.cc | 19 +- .../src/api_manager/utils/url_util.h | 2 + 13 files changed, 1169 insertions(+), 283 deletions(-) create mode 100644 contrib/endpoints/src/api_manager/firebase_rules/BUILD create mode 100644 contrib/endpoints/src/api_manager/firebase_rules/firebase_request.cc create mode 100644 contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h diff --git a/contrib/endpoints/src/api_manager/BUILD b/contrib/endpoints/src/api_manager/BUILD index 39e85eacac5..83351e10a55 100644 --- a/contrib/endpoints/src/api_manager/BUILD +++ b/contrib/endpoints/src/api_manager/BUILD @@ -118,6 +118,7 @@ cc_library( "//contrib/endpoints/src/api_manager/context", "//contrib/endpoints/src/api_manager/service_control", "//contrib/endpoints/src/api_manager/utils", + "//contrib/endpoints/src/api_manager/firebase_rules", "//external:cc_wkt_protos", "//external:cloud_trace", "//external:googletest_prod", @@ -291,6 +292,8 @@ cc_test( deps = [ ":api_manager", ":mock_api_manager_environment", + ":security_rules_proto", + "//external:cc_wkt_protos", "//external:googletest_main", ], ) diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token.h b/contrib/endpoints/src/api_manager/auth/service_account_token.h index 51e99c65b89..3a3453daf91 100644 --- a/contrib/endpoints/src/api_manager/auth/service_account_token.h +++ b/contrib/endpoints/src/api_manager/auth/service_account_token.h @@ -65,6 +65,9 @@ class ServiceAccountToken { JWT_TOKEN_FOR_SERVICE_CONTROL = 0, JWT_TOKEN_FOR_CLOUD_TRACING, JWT_TOKEN_FOR_FIREBASE, + + // JWT token for accessing the http endpoints defined in Firebase Rules. + JWT_TOKEN_FOR_AUTHORIZATION_SERVICE, JWT_TOKEN_TYPE_MAX, }; // Set audience. Only calcualtes JWT token with specified audience. diff --git a/contrib/endpoints/src/api_manager/check_security_rules.cc b/contrib/endpoints/src/api_manager/check_security_rules.cc index 845bc4cf6bb..2bb5180e981 100644 --- a/contrib/endpoints/src/api_manager/check_security_rules.cc +++ b/contrib/endpoints/src/api_manager/check_security_rules.cc @@ -17,87 +17,41 @@ #include #include #include "contrib/endpoints/src/api_manager/auth/lib/json_util.h" -#include "contrib/endpoints/src/api_manager/proto/security_rules.pb.h" +#include "contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h" #include "contrib/endpoints/src/api_manager/utils/marshalling.h" -using ::google::api_manager::auth::GetProperty; using ::google::api_manager::auth::GetStringValue; +using ::google::api_manager::firebase_rules::FirebaseRequest; using ::google::api_manager::utils::Status; -using ::google::protobuf::Map; -using ::google::protobuf::util::error::Code; namespace google { namespace api_manager { namespace { -const char kFailedFirebaseReleaseFetch[] = "Failed to fetch Firebase Release"; -const char kFailedFirebaseTest[] = "Failed to execute Firebase Test"; -const char kInvalidResponse[] = "Invalid JSON response from Firebase Service"; -const char kTestSuccess[] = "SUCCESS"; -const char kHttpGetMethod[] = "GET"; -const char kHttpPostMethod[] = "POST"; -const char kHttpHeadMethod[] = "HEAD"; -const char kHttpOptionsMethod[] = "OPTIONS"; -const char kHttpDeleteMethod[] = "DELETE"; -const char kFirebaseCreateMethod[] = "create"; -const char kFirebaseGetMethod[] = "get"; -const char kFirebaseDeleteMethod[] = "delete"; -const char kFirebaseUpdateMethod[] = "update"; -const char kV1[] = "/v1"; -const char kTestQuery[] = ":test?alt=json"; -const char kProjects[] = "/projects"; -const char kReleases[] = "/releases"; -const char kRulesetName[] = "rulesetName"; -const char kTestResults[] = "testResults"; -const char kState[] = "state"; -const char kToken[] = "token"; -const char kAuth[] = "auth"; -const char kRequest[] = "request"; -const char kContentType[] = "Content-Type"; -const char kApplication[] = "application/json"; - -void SetProtoValue(const std::string &key, - const ::google::protobuf::Value &value, - ::google::protobuf::Value *head) { - ::google::protobuf::Struct *s = head->mutable_struct_value(); - Map *fields = s->mutable_fields(); - (*fields)[key] = value; -} +const std::string kFailedFirebaseReleaseFetch = + "Failed to fetch Firebase Release"; +const std::string kFailedFirebaseTest = "Failed to execute Firebase Test"; +const std::string kInvalidResponse = + "Invalid JSON response from Firebase Service"; +const std::string kV1 = "/v1"; +const std::string kHttpGetMethod = "GET"; +const std::string kProjects = "/projects"; +const std::string kReleases = "/releases"; +const std::string kRulesetName = "rulesetName"; +const std::string kContentType = "Content-Type"; +const std::string kApplication = "application/json"; std::string GetReleaseName(const context::RequestContext &context) { return context.service_context()->service_name() + ":" + context.service_context()->service().apis(0).version(); } -std::string GetRulesetTestUri(const context::RequestContext &context, - const std::string &ruleset_id) { - return context.service_context()->config()->GetFirebaseServer() + kV1 + "/" + - ruleset_id + kTestQuery; -} - std::string GetReleaseUrl(const context::RequestContext &context) { return context.service_context()->config()->GetFirebaseServer() + kV1 + kProjects + "/" + context.service_context()->project_id() + kReleases + "/" + GetReleaseName(context); } -std::string GetOperation(const std::string &httpMethod) { - if (httpMethod == kHttpPostMethod) { - return kFirebaseCreateMethod; - } - - if (httpMethod == kHttpGetMethod || httpMethod == kHttpHeadMethod || - httpMethod == kHttpOptionsMethod) { - return kFirebaseGetMethod; - } - - if (httpMethod == kHttpDeleteMethod) { - return kFirebaseDeleteMethod; - } - - return kFirebaseUpdateMethod; -} - // An AuthzChecker object is created for every incoming request. It does // authorizaiton by calling Firebase Rules service. class AuthzChecker : public std::enable_shared_from_this { @@ -111,38 +65,25 @@ class AuthzChecker : public std::enable_shared_from_this { std::function continuation); private: - // Helper method that invokes the test firebase service api. - void CallTest(const std::string &ruleset_id, - std::shared_ptr context, - std::function continuation); + // This method invokes the Firebase TestRuleset API endpoint as well as user + // defined endpoints provided by the TestRulesetResponse. + void CallNextRequest(std::function continuation); - // Parse the respose for GET RELEASE API call + // Parse the response for GET RELEASE API call Status ParseReleaseResponse(const std::string &json_str, std::string *ruleset_id); - // Parses the response for the TEST API call - Status ParseTestResponse(context::RequestContext &context, - const std::string &json_str); - - // Builds the request body for the TESP API call. - Status BuildTestRequestBody(context::RequestContext &context, - std::string *result_string); - // Invoke the HTTP call void HttpFetch(const std::string &url, const std::string &method, const std::string &request_body, + auth::ServiceAccountToken::JWT_TOKEN_TYPE token_type, std::function continuation); - // Get the auth token for Firebase service - const std::string &GetAuthToken() { - return sa_token_->GetAuthToken( - auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE); - } - std::shared_ptr GetPtr() { return shared_from_this(); } ApiManagerEnvInterface *env_; auth::ServiceAccountToken *sa_token_; + std::unique_ptr request_handler_; }; AuthzChecker::AuthzChecker(ApiManagerEnvInterface *env, @@ -162,9 +103,10 @@ void AuthzChecker::Check( return; } - // Fetch the Release attributes. + // Fetch the Release attributes and get ruleset name. auto checker = GetPtr(); HttpFetch(GetReleaseUrl(*context), kHttpGetMethod, "", + auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE, [context, final_continuation, checker](Status status, std::string &&body) { std::string ruleset_id; @@ -182,39 +124,38 @@ void AuthzChecker::Check( // If the parsing of the release body is successful, then call the // Test Api for firebase rules service. if (status.ok()) { - checker->CallTest(ruleset_id, context, final_continuation); + checker->request_handler_ = std::unique_ptr( + new FirebaseRequest(ruleset_id, checker->env_, context)); + checker->CallNextRequest(final_continuation); } else { final_continuation(status); } }); } -void AuthzChecker::CallTest(const std::string &ruleset_id, - std::shared_ptr context, - std::function continuation) { - std::string body; - Status status = BuildTestRequestBody(*context.get(), &body); - if (!status.ok()) { - continuation(status); +void AuthzChecker::CallNextRequest( + std::function continuation) { + if (request_handler_->is_done()) { + continuation(request_handler_->RequestStatus()); return; } auto checker = GetPtr(); - HttpFetch(GetRulesetTestUri(*context, ruleset_id), kHttpPostMethod, body, - [context, continuation, checker, ruleset_id](Status status, - std::string &&body) { + firebase_rules::HttpRequest http_request = request_handler_->GetHttpRequest(); + HttpFetch(http_request.url, http_request.method, http_request.body, + http_request.token_type, + [continuation, checker](Status status, std::string &&body) { + checker->env_->LogError(std::string("Response Body = ") + body); if (status.ok()) { - checker->env_->LogDebug( - std::string("Test API succeeded with ") + body); - status = checker->ParseTestResponse(*context.get(), body); + checker->request_handler_->UpdateResponse(body); + checker->CallNextRequest(continuation); } else { checker->env_->LogError(std::string("Test API failed with ") + status.ToString()); status = Status(Code::INTERNAL, kFailedFirebaseTest); + continuation(status); } - - continuation(status); }); } @@ -228,7 +169,7 @@ Status AuthzChecker::ParseReleaseResponse(const std::string &json_str, } Status status = Status::OK; - const char *id = GetStringValue(json, kRulesetName); + const char *id = GetStringValue(json, kRulesetName.c_str()); *ruleset_id = (id == nullptr) ? "" : id; if (ruleset_id->empty()) { @@ -242,85 +183,10 @@ Status AuthzChecker::ParseReleaseResponse(const std::string &json_str, return status; } -Status AuthzChecker::ParseTestResponse(context::RequestContext &context, - const std::string &json_str) { - grpc_json *json = grpc_json_parse_string_with_len( - const_cast(json_str.data()), json_str.length()); - - if (!json) { - return Status(Code::INVALID_ARGUMENT, - "Invalid JSON response from Firebase Service"); - } - - Status status = Status::OK; - Status invalid = Status(Code::INTERNAL, kInvalidResponse); - - const grpc_json *testResults = GetProperty(json, kTestResults); - if (testResults == nullptr) { - env_->LogError("TestResults are null"); - status = invalid; - } else { - const char *result = GetStringValue(testResults->child, kState); - if (result == nullptr) { - env_->LogInfo("Result state is empty"); - status = invalid; - } else if (std::string(result) != kTestSuccess) { - status = Status(Code::PERMISSION_DENIED, - std::string("Unauthorized ") + - context.request()->GetRequestHTTPMethod() + - " access to resource " + - context.request()->GetRequestPath(), - Status::AUTH); - } - } - - grpc_json_destroy(json); - return status; -} - -Status AuthzChecker::BuildTestRequestBody(context::RequestContext &context, - std::string *result_string) { - proto::TestRulesetRequest request; - auto *test_case = request.add_test_cases(); - auto httpMethod = context.request()->GetRequestHTTPMethod(); - - test_case->set_service_name(context.service_context()->service_name()); - test_case->set_resource_path(context.request()->GetRequestPath()); - test_case->set_operation(GetOperation(httpMethod)); - test_case->set_expectation(proto::TestRulesetRequest::TestCase::ALLOW); - - ::google::protobuf::Value auth; - ::google::protobuf::Value token; - ::google::protobuf::Value claims; - - Status status = utils::JsonToProto(context.auth_claims(), &claims); - if (!status.ok()) { - env_->LogError(std::string("Error creating Protobuf from claims") + - status.ToString()); - return status; - } - - SetProtoValue(kToken, claims, &token); - SetProtoValue(kAuth, token, &auth); - - auto *variables = test_case->mutable_variables(); - (*variables)[kRequest] = auth; - - status = - utils::ProtoToJson(request, result_string, utils::JsonOptions::DEFAULT); - if (status.ok()) { - env_->LogDebug(std::string("PRotobuf to JSON string = ") + *result_string); - } else { - env_->LogError(std::string("Error creating TestRulesetRequest") + - status.ToString()); - } - - return status; -} - void AuthzChecker::HttpFetch( const std::string &url, const std::string &method, const std::string &request_body, + auth::ServiceAccountToken::JWT_TOKEN_TYPE token_type, std::function continuation) { env_->LogDebug(std::string("Issue HTTP Request to url :") + url + " method : " + method + " body: " + request_body); @@ -334,9 +200,10 @@ void AuthzChecker::HttpFetch( return; } - request->set_method(method).set_url(url).set_auth_token(GetAuthToken()); + request->set_method(method).set_url(url).set_auth_token( + sa_token_->GetAuthToken(token_type)); - if (method != kHttpGetMethod) { + if (!request_body.empty()) { request->set_header(kContentType, kApplication).set_body(request_body); } diff --git a/contrib/endpoints/src/api_manager/check_security_rules_test.cc b/contrib/endpoints/src/api_manager/check_security_rules_test.cc index 97c555bc4df..bf60d21bd60 100644 --- a/contrib/endpoints/src/api_manager/check_security_rules_test.cc +++ b/contrib/endpoints/src/api_manager/check_security_rules_test.cc @@ -20,9 +20,13 @@ #include "contrib/endpoints/src/api_manager/context/service_context.h" #include "contrib/endpoints/src/api_manager/mock_api_manager_environment.h" #include "contrib/endpoints/src/api_manager/mock_request.h" +#include "contrib/endpoints/src/api_manager/proto/security_rules.pb.h" +#include "contrib/endpoints/src/api_manager/utils/marshalling.h" +#include "google/protobuf/util/message_differencer.h" using ::testing::_; using ::testing::AllOf; +using ::testing::InSequence; using ::testing::Invoke; using ::testing::Property; using ::testing::Return; @@ -30,8 +34,18 @@ using ::testing::StrCaseEq; using ::testing::StrEq; using ::testing::StrNe; +using ::google::protobuf::util::MessageDifferencer; using ::google::api_manager::utils::Status; +using ::google::protobuf::Map; using ::google::protobuf::util::error::Code; +using ::google::protobuf::RepeatedPtrField; + +// Tuple with arg<0> = function name +// arg<1> = url, arg<2> = method, arg<3> = body. +using FuncTuple = + std::tuple; +using ::google::api_manager::proto::TestRulesetResponse; +using FunctionCall = TestRulesetResponse::TestResult::FunctionCall; namespace google { namespace api_manager { @@ -138,27 +152,69 @@ static const char kReleaseError[] = R"( } })"; -// TestRuleset returns Failure which means unauthorized access. -static const char kTestResultFailure[] = R"( +static const char kDummyBody[] = R"( { - "testResults": [ - { - "state": "FAILURE" - } - ] + "key" : "value" +})"; + +const char kFirstRequest[] = + R"({"testSuite":{"testCases":[{"expectation":"ALLOW","request":{"method":"get","path":"/ListShelves","auth":{"token":{"email":"limin-429@appspot.gserviceaccount.com","email_verified":true,"azp":"limin-429@appspot.gserviceaccount.com","aud":"https://myfirebaseapp.appspot.com","sub":"113424383671131376652","iat":1486575396,"iss":"https://accounts.google.com","exp":1486578996}}}}]}})"; + +const char kSecondRequest[] = + R"({"testSuite":{"testCases":[{"expectation":"ALLOW","request":{"auth":{"token":{"email":"limin-429@appspot.gserviceaccount.com","azp":"limin-429@appspot.gserviceaccount.com","aud":"https://myfirebaseapp.appspot.com","sub":"113424383671131376652","iss":"https://accounts.google.com","email_verified":true,"iat":1486575396,"exp":1486578996}},"method":"get","path":"/ListShelves"},"functionMocks":[{"function":"f1","args":[{"exactValue":"http://url1"},{"exactValue":"POST"},{"exactValue":{"key":"value"}}],"result":{"value":{"key":"value"}}}]}]}})"; + +const char kThirdRequest[] = + R"({"testSuite":{"testCases":[{"expectation":"ALLOW","request":{"method":"get","path":"/ListShelves","auth":{"token":{"email":"limin-429@appspot.gserviceaccount.com","iat":1486575396,"azp":"limin-429@appspot.gserviceaccount.com","exp":1486578996,"email_verified":true,"sub":"113424383671131376652","aud":"https://myfirebaseapp.appspot.com","iss":"https://accounts.google.com"}}},"functionMocks":[{"function":"f2","args":[{"exactValue":"http://url2"},{"exactValue":"GET"},{"exactValue":{"key":"value"}}],"result":{"value":{"key":"value"}}},{"function":"f3","args":[{"exactValue":"https://url3"},{"exactValue":"GET"},{"exactValue":{"key":"value"}}],"result":{"value":{"key":"value"}}},{"function":"f1","args":[{"exactValue":"http://url1"},{"exactValue":"POST"},{"exactValue":{"key":"value"}}],"result":{"value":{"key":"value"}}}]}]}})"; + +::google::protobuf::Value ToValue(const std::string &arg) { + ::google::protobuf::Value value; + value.set_string_value(arg); + return value; } -)"; -// TestRuleset call to Firebase response on success. -static const char kTestResultSuccess[] = R"( -{ - "testResults": [ - { - "state": "SUCCESS" - } - ] +MATCHER_P3(HTTPRequestMatches, url, method, body, "") { + if (arg->url() != url) { + return false; + } + + if (strcasecmp(method.c_str(), arg->method().c_str()) != 0) { + return false; + } + + if (body.empty() || arg->body().empty()) { + return body.empty() && arg->body().empty(); + } + + google::protobuf::Value actual; + google::protobuf::Value expected; + + if (utils::JsonToProto(body, &expected) != Status::OK || + utils::JsonToProto(arg->body(), &actual) != Status::OK) { + return false; + } + + return MessageDifferencer::Equals(actual, expected); +} + +FunctionCall BuildCall(const std::string &name, const std::string &url, + const std::string &method, const std::string &body) { + FunctionCall func_call; + func_call.set_function(name); + + if (!url.empty()) { + *(func_call.add_args()) = ToValue(url); + } + + if (!method.empty()) { + *(func_call.add_args()) = ToValue(method); + } + + if (!body.empty()) { + *(func_call.add_args()) = ToValue(body); + } + + return func_call; } -)"; // Get a server configuration that has auth disabled. This should disable // security rules check by default. @@ -295,6 +351,56 @@ class CheckSecurityRulesTest : public ::testing::Test { ":test?alt=json"; } + void ExpectCall(std::string url, std::string method, std::string body, + std::string response, Status status = Status::OK) { + EXPECT_CALL(*raw_env_, + DoRunHTTPRequest(HTTPRequestMatches(url, method, body))) + .WillOnce(Invoke([response, status](HTTPRequest *req) { + std::map empty; + std::string body(response); + req->OnComplete(status, std::move(empty), std::move(body)); + })); + } + + std::string BuildTestRulesetResponse( + bool isSuccess, std::vector funcs = std::vector()) { + TestRulesetResponse response; + auto *result = response.add_test_results(); + result->set_state(isSuccess ? TestRulesetResponse::TestResult::SUCCESS + : TestRulesetResponse::TestResult::FAILURE); + + std::string url, method, body; + for (auto http : funcs) { + auto *func = result->add_function_calls(); + func->set_function(std::get<0>(http)); + if (!std::get<1>(http).empty()) { + func->add_args()->set_string_value(std::get<1>(http)); + } + + if (!std::get<2>(http).empty()) { + func->add_args()->set_string_value(std::get<2>(http)); + } + + if (!std::get<3>(http).empty()) { + ::google::protobuf::Value body; + Status status = utils::JsonToProto(std::get<3>(http), &body); + *(func->add_args()) = body; + } + } + + std::string json_str; + utils::ProtoToJson(response, &json_str, utils::JsonOptions::DEFAULT); + return json_str; + } + + void SetProtoValue(const std::string &key, + const ::google::protobuf::Value &value, + ::google::protobuf::Value *head) { + ::google::protobuf::Struct *s = head->mutable_struct_value(); + auto *fields = s->mutable_fields(); + (*fields)[key] = value; + } + MockApiManagerEnvironment *raw_env_; MockRequest *raw_request_; std::shared_ptr request_context_; @@ -353,27 +459,9 @@ TEST_F(CheckSecurityRulesTest, CheckAuthzFailTestRuleset) { SetUp(service_config, server_config); request_context_->set_auth_claims(kJwtEmailPayload); - EXPECT_CALL(*raw_env_, DoRunHTTPRequest(AllOf( - Property(&HTTPRequest::url, StrEq(release_url_)), - Property(&HTTPRequest::method, StrCaseEq("GET"))))) - .WillOnce(Invoke([](HTTPRequest *req) { - - std::map empty; - std::string body(kRelease); - req->OnComplete(Status::OK, std::move(empty), std::move(body)); - - })); - - EXPECT_CALL(*raw_env_, - DoRunHTTPRequest( - AllOf(Property(&HTTPRequest::url, StrEq(ruleset_test_url_)), - Property(&HTTPRequest::method, StrCaseEq("POST"))))) - .WillOnce(Invoke([](HTTPRequest *req) { - std::map empty; - std::string body; - req->OnComplete(Status(Code::INTERNAL, "Cannot talk to server"), - std::move(empty), std::move(body)); - })); + ExpectCall(release_url_, "GET", "", kRelease); + ExpectCall(ruleset_test_url_, "POST", kFirstRequest, "", + Status(Code::INTERNAL, "Cannot talk to server")); CheckSecurityRules(request_context_, [](Status status) { ASSERT_TRUE(status.CanonicalCode() == Code::INTERNAL); @@ -393,26 +481,10 @@ TEST_F(CheckSecurityRulesTest, CheckAuthzFailWithTestResultFailure) { SetUp(service_config, server_config); request_context_->set_auth_claims(kJwtEmailPayload); - EXPECT_CALL(*raw_env_, DoRunHTTPRequest(AllOf( - Property(&HTTPRequest::url, StrEq(release_url_)), - Property(&HTTPRequest::method, StrCaseEq("GET"))))) - .WillOnce(Invoke([](HTTPRequest *req) { + ExpectCall(release_url_, "GET", "", kRelease); - std::map empty; - std::string body(kRelease); - req->OnComplete(Status::OK, std::move(empty), std::move(body)); - - })); - - EXPECT_CALL(*raw_env_, - DoRunHTTPRequest( - AllOf(Property(&HTTPRequest::url, StrEq(ruleset_test_url_)), - Property(&HTTPRequest::method, StrCaseEq("POST"))))) - .WillOnce(Invoke([](HTTPRequest *req) { - std::map empty; - std::string body = kTestResultFailure; - req->OnComplete(Status::OK, std::move(empty), std::move(body)); - })); + ExpectCall(ruleset_test_url_, "POST", kFirstRequest, + BuildTestRulesetResponse(false)); CheckSecurityRules(request_context_, [](Status status) { ASSERT_TRUE(status.CanonicalCode() == Code::PERMISSION_DENIED); @@ -432,30 +504,125 @@ TEST_F(CheckSecurityRulesTest, CheckAuthzSuccess) { SetUp(service_config, server_config); request_context_->set_auth_claims(kJwtEmailPayload); - EXPECT_CALL(*raw_env_, DoRunHTTPRequest(AllOf( - Property(&HTTPRequest::url, StrEq(release_url_)), - Property(&HTTPRequest::method, StrCaseEq("GET"))))) - .WillOnce(Invoke([](HTTPRequest *req) { + ExpectCall(release_url_, "GET", "", kRelease); + ExpectCall(ruleset_test_url_, "POST", kFirstRequest, + BuildTestRulesetResponse(true)); - std::map empty; - std::string body(kRelease); - req->OnComplete(Status::OK, std::move(empty), std::move(body)); + CheckSecurityRules(request_context_, + [](Status status) { ASSERT_TRUE(status.ok()); }); +} - })); +class CheckSecurityRulesFunctions : public CheckSecurityRulesTest, + public ::testing::WithParamInterface { + public: + void SetUp() { + std::string service_config = std::string(kServiceName) + + kProducerProjectId + kApis + kAuthentication + + kHttp; + std::string server_config = kServerConfig; + CheckSecurityRulesTest::SetUp(service_config, server_config); + request_context_->set_auth_claims(kJwtEmailPayload); + + InSequence s; + + ExpectCall(release_url_, "GET", "", kRelease); + ExpectCall( + ruleset_test_url_, "POST", kFirstRequest, + BuildTestRulesetResponse( + false, {std::make_tuple("f1", "http://url1", "POST", kDummyBody)})); + + ExpectCall("http://url1", "POST", kDummyBody, kDummyBody); + ExpectCall( + ruleset_test_url_, "POST", kSecondRequest, + BuildTestRulesetResponse( + false, {std::make_tuple("f2", "http://url2", "GET", kDummyBody), + std::make_tuple("f3", "https://url3", "GET", kDummyBody), + std::make_tuple("f1", "http://url1", "POST", kDummyBody)})); + ExpectCall("http://url2", "GET", kDummyBody, kDummyBody); + ExpectCall("https://url3", "GET", kDummyBody, kDummyBody); + ExpectCall(ruleset_test_url_, "POST", kThirdRequest, + BuildTestRulesetResponse( + GetParam(), + {std::make_tuple("f2", "http://url2", "GET", kDummyBody), + std::make_tuple("f3", "https://url3", "GET", kDummyBody), + std::make_tuple("f1", "http://url1", "POST", kDummyBody)})); + } +}; - EXPECT_CALL(*raw_env_, - DoRunHTTPRequest( - AllOf(Property(&HTTPRequest::url, StrEq(ruleset_test_url_)), - Property(&HTTPRequest::method, StrCaseEq("POST"))))) - .WillOnce(Invoke([](HTTPRequest *req) { - std::map empty; - std::string body = kTestResultSuccess; - req->OnComplete(Status::OK, std::move(empty), std::move(body)); - })); +// Check the function call request response loop: +// 1. ESP Send TestRulesetRequest (No function calls) +// 2. ESP gets TestRulesetResponse with f1("http://url1, "POST", kDummyBody) +// 3. ESP Send HTTP request to f1.url +// 4. ESP Send TestRulesetRequest with (f1.url, "POST", kDummyBody, +// kDummyResponse); +// 5. EST gets TestRulesetResponse with f1, f2, f3 +// 6. ESP Send HTTP request to f2.url and f3.url. (checks f1 response is +// buffered). +// 7. ESP Send TestRulesetRequest with (f1, f2, f3) +// 8. ESP receives TestRulesetRequest +TEST_P(CheckSecurityRulesFunctions, CheckMultipleFunctions) { + auto ptr = this; + auto success = GetParam(); + CheckSecurityRules(request_context_, [ptr, success](Status status) { + if (success) { + ASSERT_TRUE(status.ok()) << status.ToString(); + } else { + ASSERT_TRUE(status.CanonicalCode() == Code::PERMISSION_DENIED) + << status.ToString(); + } + }); +} - CheckSecurityRules(request_context_, - [](Status status) { ASSERT_TRUE(status.ok()); }); +INSTANTIATE_TEST_CASE_P(CheckMultipleFunctionSuccessFailure, + CheckSecurityRulesFunctions, ::testing::Bool()); + +class CheckSecurityRulesBadFunctions + : public CheckSecurityRulesTest, + public ::testing::WithParamInterface { + public: + void SetUp() { + std::string service_config = std::string(kServiceName) + + kProducerProjectId + kApis + kAuthentication + + kHttp; + std::string server_config = kServerConfig; + CheckSecurityRulesTest::SetUp(service_config, server_config); + request_context_->set_auth_claims(kJwtEmailPayload); + + InSequence s; + + ExpectCall(release_url_, "GET", "", kRelease); + ExpectCall(ruleset_test_url_, "POST", kFirstRequest, + BuildTestRulesetResponse(false, {GetParam()})); + + EXPECT_CALL(*raw_env_, + DoRunHTTPRequest(AllOf( + Property(&HTTPRequest::url, StrEq(std::get<1>(GetParam()))), + Property(&HTTPRequest::method, + StrCaseEq(std::get<2>(GetParam())))))) + .Times(0); + } +}; + +TEST_P(CheckSecurityRulesBadFunctions, CheckBadFunctionArguments) { + auto ptr = this; + CheckSecurityRules(request_context_, [ptr](Status status) { + ASSERT_TRUE(status.CanonicalCode() == Code::INVALID_ARGUMENT) + << status.ToString(); + }); } + +INSTANTIATE_TEST_CASE_P(CheckSecurityRulesBadFunctionArguments, + CheckSecurityRulesBadFunctions, + ::testing::Values( + // Empty function name + std::make_tuple("", "http://url1", "POST", + kDummyBody), + // Argument count less than 2 + std::make_tuple("f1", "", "", ""), + // The url is not set + std::make_tuple("f1", "", "POST", kDummyBody), + // The url is not a http or https protocol + std::make_tuple("f1", "ftp://url1", "BODY", ""))); } } // namespace api_manager } // namespace google diff --git a/contrib/endpoints/src/api_manager/context/request_context.h b/contrib/endpoints/src/api_manager/context/request_context.h index c633952a811..57706c27a2e 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.h +++ b/contrib/endpoints/src/api_manager/context/request_context.h @@ -42,7 +42,7 @@ class RequestContext { } // Get the request object. - Request *request() { return request_.get(); } + Request *request() const { return request_.get(); } // Get the method info. const MethodInfo *method() const { return method_call_.method_info; } @@ -116,7 +116,7 @@ class RequestContext { void set_auth_claims(const std::string &claims) { auth_claims_ = claims; } - const std::string &auth_claims() { return auth_claims_; } + const std::string &auth_claims() const { return auth_claims_; } private: // Fill OperationInfo diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index 28ce9271f80..8e1493d3cfa 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -76,6 +76,18 @@ ServiceContext::ServiceContext(std::unique_ptr env, service_account_token_.SetAudience( auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE, kFirebaseAudience); + + if (config_->server_config() && + !config_->server_config() + ->api_check_security_rules_config() + .authorization_service_audience() + .empty()) { + service_account_token_.SetAudience( + auth::ServiceAccountToken::JWT_TOKEN_FOR_AUTHORIZATION_SERVICE, + config_->server_config() + ->api_check_security_rules_config() + .authorization_service_audience()); + } } MethodCallInfo ServiceContext::GetMethodCallInfo( diff --git a/contrib/endpoints/src/api_manager/firebase_rules/BUILD b/contrib/endpoints/src/api_manager/firebase_rules/BUILD new file mode 100644 index 00000000000..8edaaba9411 --- /dev/null +++ b/contrib/endpoints/src/api_manager/firebase_rules/BUILD @@ -0,0 +1,41 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +package(default_visibility = ["//contrib/endpoints/src/api_manager:__subpackages__"]) + +cc_library( + name = "firebase_rules", + srcs = [ + "firebase_request.cc", + ], + hdrs = [ + "firebase_request.h", + ], + linkopts = select({ + "//:darwin": [], + "//conditions:default": [ + "-lm", + "-luuid", + ], + }), + deps = [ + "//contrib/endpoints/src/api_manager:security_rules_proto", + "//contrib/endpoints/src/api_manager/context", + "//contrib/endpoints/src/api_manager/utils", + "//external:cc_wkt_protos", + "//external:googletest_prod", + ], +) diff --git a/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.cc b/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.cc new file mode 100644 index 00000000000..300d521ed1f --- /dev/null +++ b/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.cc @@ -0,0 +1,403 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +#include "contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h" +#include "contrib/endpoints/src/api_manager/utils/marshalling.h" +#include "contrib/endpoints/src/api_manager/utils/url_util.h" +#include "google/protobuf/util/message_differencer.h" + +#include +#include +using ::google::api_manager::utils::Status; +using ::google::api_manager::proto::TestRulesetResponse; +using ::google::protobuf::util::MessageDifferencer; +using ::google::protobuf::Map; +using TestRulesetResponse = ::google::api_manager::proto::TestRulesetResponse; +using FunctionCall = TestRulesetResponse::TestResult::FunctionCall; +using ::google::protobuf::RepeatedPtrField; + +namespace google { +namespace api_manager { +namespace firebase_rules { + +namespace { + +const std::string kToken = "token"; +const std::string kAuth = "auth"; +const std::string kPath = "path"; +const std::string kMethod = "method"; +const std::string kHttpGetMethod = "GET"; +const std::string kHttpPostMethod = "POST"; +const std::string kHttpHeadMethod = "HEAD"; +const std::string kHttpOptionsMethod = "OPTIONS"; +const std::string kHttpDeleteMethod = "DELETE"; +const std::string kFirebaseCreateMethod = "create"; +const std::string kFirebaseGetMethod = "get"; +const std::string kFirebaseDeleteMethod = "delete"; +const std::string kFirebaseUpdateMethod = "update"; +const std::string kV1 = "/v1"; +const std::string kTestQuery = ":test?alt=json"; + +void SetProtoValue(const std::string &key, + const ::google::protobuf::Value &value, + ::google::protobuf::Value *head) { + ::google::protobuf::Struct *s = head->mutable_struct_value(); + Map *fields = s->mutable_fields(); + (*fields)[key] = value; +} + +// Convert HTTP method to Firebase specific method. +const std::string &GetOperation(const std::string &httpMethod) { + if (httpMethod == kHttpPostMethod) { + return kFirebaseCreateMethod; + } + + if (httpMethod == kHttpGetMethod || httpMethod == kHttpHeadMethod || + httpMethod == kHttpOptionsMethod) { + return kFirebaseGetMethod; + } + + if (httpMethod == kHttpDeleteMethod) { + return kFirebaseDeleteMethod; + } + + return kFirebaseUpdateMethod; +} +} + +// Constructor +FirebaseRequest::FirebaseRequest( + const std::string &ruleset_name, ApiManagerEnvInterface *env, + std::shared_ptr context) + : env_(env), + context_(context), + ruleset_name_(ruleset_name), + firebase_server_( + context->service_context()->config()->GetFirebaseServer()), + current_status_(Status::OK), + is_done_(false), + next_request_(nullptr) { + firebase_http_request_.url = + firebase_server_ + kV1 + "/" + ruleset_name + kTestQuery; + firebase_http_request_.method = kHttpPostMethod; + firebase_http_request_.token_type = + auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE; + external_http_request_.token_type = + auth::ServiceAccountToken::JWT_TOKEN_FOR_AUTHORIZATION_SERVICE; + + // Update the first request to be sent which is the TestRulesetRequest + // request. + SetStatus(UpdateRulesetRequestBody(RepeatedPtrField())); + if (!current_status_.ok()) { + return; + } + + next_request_ = &firebase_http_request_; +} + +bool FirebaseRequest::is_done() { return is_done_; } + +HttpRequest FirebaseRequest::GetHttpRequest() { + if (is_done()) { + return HttpRequest(); + } + + if (next_request_ == nullptr) { + SetStatus(Status(Code::INTERNAL, "Internal state in error")); + return HttpRequest(); + } + + return *next_request_; +} + +Status FirebaseRequest::RequestStatus() { return current_status_; } + +void FirebaseRequest::UpdateResponse(const std::string &body) { + GOOGLE_DCHECK(!is_done()) + << "Receive a response body when no HTTP request is outstanding"; + + GOOGLE_DCHECK(next_request_) + << "Received a response when there is no request set" + "and when is_done is false." + " Looks like a code bug..."; + + if (is_done() || next_request_ == nullptr) { + SetStatus(Status(Code::INTERNAL, + "Internal state error while processing Http request")); + return; + } + + Status status = Status::OK; + + // If the previous request was firebase request, then process its response. + // Otherwise, it is the response for external HTTP request. + if (next_request_ == &firebase_http_request_) { + status = ProcessTestRulesetResponse(body); + } else { + status = ProcessFunctionCallResponse(body); + } + + if (status.ok()) { + status = SetNextRequest(); + } + + SetStatus(status); + return; +} + +void FirebaseRequest::SetStatus(const Status &status) { + if (!status.ok() && !is_done_) { + current_status_ = status; + is_done_ = true; + } +} + +// Create the TestRulesetRequest body. +Status FirebaseRequest::UpdateRulesetRequestBody( + const RepeatedPtrField &function_calls) { + proto::TestRulesetRequest request; + auto test_case = request.mutable_test_suite()->add_test_cases(); + test_case->set_expectation(proto::TestCase::ALLOW); + + ::google::protobuf::Value token; + ::google::protobuf::Value claims; + ::google::protobuf::Value path; + ::google::protobuf::Value method; + + Status status = utils::JsonToProto(context_->auth_claims(), &claims); + if (!status.ok()) { + return status; + } + + auto *variables = test_case->mutable_request()->mutable_struct_value(); + auto *fields = variables->mutable_fields(); + + path.set_string_value(context_->request()->GetRequestPath()); + (*fields)[kPath] = path; + + method.set_string_value( + GetOperation(context_->request()->GetRequestHTTPMethod())); + (*fields)[kMethod] = method; + + SetProtoValue(kToken, claims, &token); + (*fields)[kAuth] = token; + + for (auto func_call : function_calls) { + status = AddFunctionMock(&request, func_call); + if (!status.ok()) { + return status; + } + } + + std::string body; + status = utils::ProtoToJson(request, &body, utils::JsonOptions::DEFAULT); + if (status.ok()) { + env_->LogDebug(std::string("FIREBASE REQUEST BODY = ") + body); + firebase_http_request_.body = body; + } + + return status; +} + +Status FirebaseRequest::ProcessTestRulesetResponse(const std::string &body) { + Status status = utils::JsonToProto(body, &response_); + if (!status.ok()) { + return status; + } + + // If the state is SUCCESS, then we don't need to do any further processing. + if (response_.test_results(0).state() == + TestRulesetResponse::TestResult::SUCCESS) { + is_done_ = true; + next_request_ = nullptr; + return Status::OK; + } + + // Check that the test results size is 1 since we always send a single test + // case. + if (response_.test_results_size() != 1) { + std::ostringstream oss; + oss << "Received TestResultsetResponse with size = " + << response_.test_results_size() << " expecting only 1 test result"; + + env_->LogError(oss.str()); + return Status(Code::INTERNAL, "Unexpected TestResultsetResponse"); + } + + bool allFunctionsProcessed = true; + + // Iterate over all the function calls and make sure that the function calls + // are well formed. + for (auto func_call : response_.test_results(0).function_calls()) { + status = CheckFuncCallArgs(func_call); + if (!status.ok()) { + return status; + } + allFunctionsProcessed &= Find(func_call) != funcs_with_result_.end(); + } + + // Since all the functions have a response and the state is FAILURE, this + // means Unauthorized access to the resource. + if (allFunctionsProcessed) { + std::string message = "Unauthorized Access"; + if (response_.test_results(0).debug_messages_size() > 0) { + std::ostringstream oss; + for (std::string msg : response_.test_results(0).debug_messages()) { + oss << msg << " "; + } + message = oss.str(); + } + + return Status(Code::PERMISSION_DENIED, message); + } + + func_call_iter_ = response_.test_results(0).function_calls().begin(); + return Status::OK; +} + +std::vector>::const_iterator +FirebaseRequest::Find(const FunctionCall &func_call) { + return std::find_if(funcs_with_result_.begin(), funcs_with_result_.end(), + [func_call](std::tuple item) { + return MessageDifferencer::Equals(std::get<0>(item), + func_call); + }); +} + +Status FirebaseRequest::ProcessFunctionCallResponse(const std::string &body) { + if (is_done() || AllFunctionCallsProcessed()) { + return Status(Code::INTERNAL, + "No external function calls present." + " But received a response. Possible code bug"); + } + + funcs_with_result_.emplace_back(*func_call_iter_, body); + func_call_iter_++; + return Status::OK; +} + +// Sets the next HTTP request that should be issued. +Status FirebaseRequest::SetNextRequest() { + if (is_done()) { + next_request_ = nullptr; + return current_status_; + } + + Status status = Status::OK; + + // While there are more functions that should be processed, check if the HTTP + // response for the function is already buffered. Set the next HTTP request if + // we find a new function and break. + while (!AllFunctionCallsProcessed()) { + if (Find(*func_call_iter_) == funcs_with_result_.end()) { + auto call = *func_call_iter_; + external_http_request_.url = call.args(0).string_value(); + external_http_request_.method = call.args(1).string_value(); + std::string body; + status = + utils::ProtoToJson(call.args(2), &body, utils::JsonOptions::DEFAULT); + if (status.ok()) { + external_http_request_.body = body; + next_request_ = &external_http_request_; + } + break; + } + + func_call_iter_++; + } + + // If All functions are processed, then issue a TestRulesetRequest. + if (AllFunctionCallsProcessed()) { + next_request_ = &firebase_http_request_; + return UpdateRulesetRequestBody(response_.test_results(0).function_calls()); + } + + return status; +} + +Status FirebaseRequest::CheckFuncCallArgs(const FunctionCall &func) { + if (func.function().empty()) { + return Status(Code::INVALID_ARGUMENT, "No function name provided"); + } + + // We only support functions that call with three argument: HTTP URL, HTTP + // method and body. The body can be empty + if (func.args_size() < 2 || func.args_size() > 3) { + std::ostringstream os; + os << func.function() << " Require 2 or 3 arguments. But has " + << func.args_size(); + return Status(Code::INVALID_ARGUMENT, os.str()); + } + + if (func.args(0).kind_case() != google::protobuf::Value::kStringValue || + func.args(1).kind_case() != google::protobuf::Value::kStringValue) { + return Status( + Code::INVALID_ARGUMENT, + std::string(func.function() + " Arguments 1 and 2 should be strings")); + } + + if (!utils::IsHttpRequest(func.args(0).string_value())) { + return Status( + Code::INVALID_ARGUMENT, + func.function() + " The first argument should be a HTTP request"); + } + + if (std::string(func.args(1).string_value()).empty()) { + return Status( + Code::INVALID_ARGUMENT, + func.function() + " argument 2 [HTTP METHOD] cannot be emtpy"); + } + + return Status::OK; +} + +bool FirebaseRequest::AllFunctionCallsProcessed() { + return func_call_iter_ == response_.test_results(0).function_calls().end(); +} + +Status FirebaseRequest::AddFunctionMock(proto::TestRulesetRequest *request, + const FunctionCall &func_call) { + if (Find(func_call) == funcs_with_result_.end()) { + return Status(Code::INTERNAL, + std::string("Cannot find body for function call") + + func_call.function()); + } + + auto *func_mock = request->mutable_test_suite() + ->mutable_test_cases(0) + ->add_function_mocks(); + + func_mock->set_function(func_call.function()); + for (auto arg : func_call.args()) { + auto *toAdd = func_mock->add_args()->mutable_exact_value(); + *toAdd = arg; + } + + ::google::protobuf::Value result_json; + Status status = + utils::JsonToProto(std::get<1>(*Find(func_call)), &result_json); + if (!status.ok()) { + env_->LogError(std::string("Error creating protobuf from request body") + + status.ToString()); + return status; + } + + *(func_mock->mutable_result()->mutable_value()) = result_json; + return Status::OK; +} + +} // namespace firebase_rules +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h b/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h new file mode 100644 index 00000000000..ee26b77e646 --- /dev/null +++ b/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h @@ -0,0 +1,197 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef FIREBASE_RULES_FIREBASE_REQUEST_H_ +#define FIREBASE_RULES_FIREBASE_REQUEST_H_ + +#include +#include +#include +#include "contrib/endpoints/include/api_manager/utils/status.h" +#include "contrib/endpoints/src/api_manager/context/request_context.h" +#include "contrib/endpoints/src/api_manager/proto/security_rules.pb.h" + +// An object of this class should be created for each RequestContext object. +// Here is the flow of messages between ESP, Firebase rules and User provided +// HTTP endpoint: +// +// 1) ESP invokes GetRelease API call on Firebase Service to get the ruleset +// name associated with the Release. The ruleset a representation of the rules +// files that the user deployed to be enforced. The release name is built by +// concatenating the service name and the api version number. +// +// 2) ESP receives the response from Firebase Service which contains the ruleset +// name associated with the Release. From this point on, ESP invokes TestRuleset +// request against this ruleset name. +// +// 3) ESP issues the TestRuleset request that includes the following +// information: +// -- The payload of the JWT token which contains, uid, email or any additional +// claims that can be used to authorize the user. +// -- A test case that provides the Request's HTTP method and HTTP path. +// Note that the above information is provided for ALL TestRuleset requests. +// +// 4) ESP receives a response in TestRulesetResponse message which has a state +// variable that is either set to SUCCESS or FAILURE. +// -- If the state is SUCCESS, then ESP considers this as authorization +// approval and invokes the continution provided with Status::OK. +// -- If the state is FAILURE, then ESP looks more into the TestRulesetResponse +// message to see if there are any user defined HTTP requests that are to be +// invoked and ESP has not seen this request before. If there are no such +// functions, then ESP stops processing with Unauthorized Access error. +// Otherwise, ESP does Step 5. +// +// 5) ESP invokes rules defined HTTP requests that are not yet seen. +// ESP invokes these requests sequentially. Once all HTTP requests are invoked, +// then ESP builds a TestRulesetRequest which contains the following in addition +// to the JWT claims and HTTP method and HTTP path. +// -- For each HTTP request, ESP converts the JSON object into a protobuf::Value +// and sets the result of the HTTP call that be accessed in the Firebase rules +// like a Map. +// ESP send the TestRuleset message to Firebase Service and processing moved to +// step 4) above. +namespace google { +namespace api_manager { +namespace firebase_rules { + +// This structure models any HTTP request that is to be invoked. These include +// both the TestRuleset Request as well as the user defined requests. +struct HttpRequest { + std::string url; + std::string method; + std::string body; + auth::ServiceAccountToken::JWT_TOKEN_TYPE token_type; +}; + +// A FirebaseRequest object understands the various http requests that need +// to be generated as a part of the TestRuleset request and response cycle. +// Here is the intented use of this code: +// FirebaseRequest request(...); +// while(!request.is_done()) { +// std::string url, method, body; +// +// /* The following is not a valid C++ statement. But written so the reader can +// get a general idea ... */ +// +// (url, method, body, token_type) = request.GetHttpRequest(); +// std::string body = InvokeHttpRequest(url, method, body, +// GetToken(token_type)); +// updateResponse(body); +// } +// +// if (request.RequestStatus.ok()) { +// .... ALLOW ..... +// } else { +// .... DENY ..... +// } +class FirebaseRequest { + public: + // Constructor. + FirebaseRequest(const std::string &ruleset_name, ApiManagerEnvInterface *env, + std::shared_ptr context); + + // If the firebase Request calling can be terminated. + bool is_done(); + + // Get the request status. This request status is only valid if is_done is + // true. + utils::Status RequestStatus(); + + // This call should be invoked to get the next http request to execute. + HttpRequest GetHttpRequest(); + + // The response for previous HttpRequest. + void UpdateResponse(const std::string &body); + + private: + utils::Status UpdateRulesetRequestBody( + const ::google::protobuf::RepeatedPtrField< + proto::TestRulesetResponse::TestResult::FunctionCall> &func_calls); + utils::Status ProcessTestRulesetResponse(const std::string &body); + utils::Status ProcessFunctionCallResponse(const std::string &body); + utils::Status CheckFuncCallArgs( + const proto::TestRulesetResponse::TestResult::FunctionCall &func); + utils::Status AddFunctionMock( + proto::TestRulesetRequest *request, + const proto::TestRulesetResponse::TestResult::FunctionCall &func_call); + void SetStatus(const utils::Status &status); + utils::Status SetNextRequest(); + bool AllFunctionCallsProcessed(); + std::vector>::const_iterator + Find(const proto::TestRulesetResponse::TestResult::FunctionCall &func_call); + + // The API manager environment. Primarily used for logging. + ApiManagerEnvInterface *env_; + + // The request context for the current request in progress. + std::shared_ptr context_; + + // The test ruleset name which contains the firebase rules and is used to + // invoke TestRuleset API. + std::string ruleset_name_; + + // The Firebase server that supports the TestRuleset requests. + std::string firebase_server_; + + // This variable tracks the status of the state machine. + utils::Status current_status_; + + // Variable to track if the state machine is done processing. This is set to + // true either when the processing is successfully done or when an error is + // encountered and current_status_ is not Statu::OK anymore. + bool is_done_; + + // The map is used to buffer the response for the user defined function calls. + std::vector> + funcs_with_result_; + + // The iterator iterates over the FunctionCalls the user wishes to invoke. So + // long as this iterator is valid, the state machine issues HTTP requests to + // the user defined HTTP endpoints. Once the iterator is equl to + // func_call_iter.end(), then the TestRuleset is issued which includes the + // function calls along with their responses. + ::google::protobuf::RepeatedPtrField< + proto::TestRulesetResponse::TestResult::FunctionCall>::const_iterator + func_call_iter_; + + // The Test ruleset response currently being processed. + proto::TestRulesetResponse response_; + + // This variable points to either firebase_http_request_ or + // external_http_request_. This will allow the UpdateResponse method to + // understand if the response received is for TestRuleset or user + // defined HTTP endpoint. If next_request points to firebase_http_request_, + // upon receiving a response, UpdateResponse will convert the response to + // TestRulesetResponse and process the response. If next_request_ points + // to external_http_request_, then the reponse provided via UpdateResponse + // is converted into a protobuf::Value. This value is initialized to nullptr + // and will be nullptr once is_done_ is set to true. + HttpRequest *next_request_; + + // The HTTP request to be sent to firebase TestRuleset API + HttpRequest firebase_http_request_; + + // The HTTP request invoked for user provided HTTP endpoint. + HttpRequest external_http_request_; +}; + +} // namespace firebase_rules +} // namespace api_manager +} // namespace google + +#endif // FIREBASE_RULES_REQUEST_HELPER_H_ diff --git a/contrib/endpoints/src/api_manager/proto/security_rules.proto b/contrib/endpoints/src/api_manager/proto/security_rules.proto index ce8d2690fe0..2c78f75f3fd 100644 --- a/contrib/endpoints/src/api_manager/proto/security_rules.proto +++ b/contrib/endpoints/src/api_manager/proto/security_rules.proto @@ -18,40 +18,223 @@ syntax = "proto3"; package google.api_manager.proto; +import "google/protobuf/empty.proto"; import "google/protobuf/struct.proto"; +// The protobufs in this file model the messages that flow from ESP to Firebase +// rules service. The naming of the protobufs start with "Test" and should not +// be confused that the protobufs are used for testing. The protobuf names and +// message structure exactly match the protobufs defined in the firebase rules. +message TestCase { + // The set of supported test case expectations. + enum Expectation { + EXPECTATION_UNSPECIFIED = 0; // Unspecified expectation. + ALLOW = 1; // Expect an allowed result. + DENY = 2; // Expect a denied result. + } + + // Mock function definition. + // + // Mocks must refer to a function declared by the target service. The type of + // the function args and result will be inferred at test time. If either the + // arg or result values are not compatible with function type declaration, the + // request will be considered invalid. + // + // More than one `FunctionMock` may be provided for a given function name so + // long as the `Arg` matchers are distinct. In the event that multiple mocks + // match the expression, the request will be treated as an invalid argument. + message FunctionMock { + // Arg matchers for the mock function. + message Arg { + // Supported argument values. + oneof type { + // Argument exactly matches value provided. + google.protobuf.Value exact_value = 1; + // Argument matches any value provided. + google.protobuf.Empty any_value = 2; + } + } + + // Possible result values from the function mock invocation. + message Result { + // Supported result values. + oneof type { + // The result is an actual value. The type of the value must match that + // of the type declared by the service. + google.protobuf.Value value = 1; + // The result is undefined, meaning the result could not be computed. + google.protobuf.Empty undefined = 2; + } + } + + // The name of the function. + // + // The function name must match one provided by a service declaration. + string function = 1; + + // The list of `Arg` values to match. The order in which the arguments are + // provided is the order in which they must appear in the function + // invocation. + repeated Arg args = 2; + + // The mock result of the function call. + Result result = 3; + } + + // Test expectation. + Expectation expectation = 1; + + // Request context. + // + // The exact format of the request context is service-dependent. See the + // appropriate service documentation for information about the supported + // fields and types on the request. Minimally, all services support the + // following fields and types: + // + // Request field | Type + // ---------------|----------------- + // auth.uid | `string` + // auth.token | `map` + // headers | `map` + // method | `string` + // params | `map` + // path | `string` + // time | `google.protobuf.Timestamp` + // + // If the request value is not well-formed for the service, the request will + // be rejected as an invalid argument. + google.protobuf.Value request = 2; + + // Optional resource value as it appears in persistent storage before the + // request is fulfilled. + // + // The resource type depends on the `request.path` value. + google.protobuf.Value resource = 3; + + // Optional function mocks for service-defined functions. If not set, any + // service defined function is expected to return an error, which may or may + // not influence the test outcome. + repeated FunctionMock function_mocks = 4; +} + +message TestSuite { + // Test cases to be executed. + repeated TestCase test_cases = 1; +} message TestRulesetRequest { - message TestCase { - // The set of supported test case expectations. - enum Expectation { - EXPECTATION_UNSPECIFIED = 0; // Unspecified expectation. - ALLOW = 1; // Expect an allowed result. - DENY = 2; // Expect a denied result. + // Name of the ruleset resource. + // Format: 'projects/{project_id}/rulesets/{ruleset_id}' + string name = 1; + + // The test suite to run against the ruleset + oneof test { + // Inline 'TestSuite' to run. + TestSuite test_suite = 3; + } +} +// Position in the `Source` content including its line, column number, and an +// index of the `File` in the `Source` message. Used for debug purposes. +message SourcePosition { + // Name of the `File`. + string file_name = 1; + + // Index of the `File` in the `Source` message where the content appears. + // @OutputOnly + int32 file_index = 2; + + // Line number of the source fragment. 1-based. + int32 line = 3; + + // First column on the source line associated with the source fragment. + int32 column = 4; + + // Position relative to the beginning of the file. This is used by the IDEA + // plugin, while the line and column are used by the compiler. + int32 current_offset = 5; +} + +message TestRulesetResponse { + // Issues include warnings, errors, and deprecation notices. + message Issue { + // The set of issue severities. + enum Severity { + // An unspecified severity. + SEVERITY_UNSPECIFIED = 0; + // Deprecation issue for statements and method that may no longer be + // supported or maintained. + DEPRECATION = 1; + // Warnings such as: unused variables. + WARNING = 2; + // Errors such as: unmatched curly braces or variable redefinition. + ERROR = 3; } - // The name of the service that is the subject of the test case. - string service_name = 1; + // Position of the issue in the `Source`. + SourcePosition source_position = 1; - // The RESTful resource path of the mock `request`. - string resource_path = 2; + // Short error description. + string description = 2; - // The `request` `operation`. The operation will typically be one of `get`, - // `list`, `create`, `update`, or `delete`. Services also may provide custom - // operations. - string operation = 3; + // The severity of the issue. + Severity severity = 3; + } + + // Test result message containing the state of the test as well as a + // description and source position for test failures. + message TestResult { + // Valid states for the test result. + enum State { + STATE_UNSPECIFIED = 0; // Test state is not set. + SUCCESS = 1; // Test is a success. + FAILURE = 2; // Test is a failure. + } + + // Represents a service-defined function call that was invoked during test + // execution. + message FunctionCall { + // Name of the function invoked. + string function = 1; - // Test expectation. - Expectation expectation = 4; + // The arguments that were provided to the function. + repeated google.protobuf.Value args = 2; + } + + // State of the test. + State state = 1; - // (-- - // Variables and fake resources need to be updated to support multiple - // services and the standardized `request` definition. - // --) + // Debug messages related to test execution issues encountered during + // evaluation. + // + // Debug messages may be related to too many or too few invocations of + // function mocks or to runtime errors that occur during evaluation. + // + // For example: ```Unable to read variable [name: "resource"]``` + repeated string debug_messages = 2; - // Optional set of variable values to use during evaluation. - map variables = 5; + // Position in the `Source` or `Ruleset` where the principle runtime error + // occurs. + // + // Evaluation of an expression may result in an error. Rules are deny by + // default, so a `DENY` expectation when an error is generated is valid. + // When there is a `DENY` with an error, the `SourcePosition` is returned. + // + // E.g. `error_position { line: 19 column: 37 }` + SourcePosition error_position = 3; + + // The set of function calls made to service-defined methods. + // + // Function calls are included in the order in which they are encountered + // during evaluation, are provided for both mocked and unmocked functions, + // and included on the response regardless of the test `state`. + repeated FunctionCall function_calls = 4; } - // The set of test cases to run against the `Source` if it is well-formed. - repeated TestCase test_cases = 3; + // Syntactic and semantic `Source` issues of varying severity. Issues of + // `ERROR` severity will prevent tests from executing. + repeated Issue issues = 1; + + // The set of test results given the test cases in the `TestSuite`. + // The results will appear in the same order as the test cases appear in the + // `TestSuite`. + repeated TestResult test_results = 2; } diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index cbee58f8da8..d3f3189fcf3 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -146,6 +146,7 @@ message ApiAuthenticationConfig { message ApiCheckSecurityRulesConfig { // Firebase server to use. string firebase_server = 1; + string authorization_service_audience = 2; } message Experimental { diff --git a/contrib/endpoints/src/api_manager/utils/url_util.cc b/contrib/endpoints/src/api_manager/utils/url_util.cc index 3d66ffa2910..06ad2e225fc 100644 --- a/contrib/endpoints/src/api_manager/utils/url_util.cc +++ b/contrib/endpoints/src/api_manager/utils/url_util.cc @@ -19,15 +19,17 @@ namespace google { namespace api_manager { namespace utils { +namespace { +const std::string kHttpPrefix = "http://"; +const std::string kHttpsPrefix = "https://"; +} std::string GetUrlContent(const std::string &url) { - static const std::string https_prefix = "https://"; - static const std::string http_prefix = "http://"; std::string result; - if (url.compare(0, https_prefix.size(), https_prefix) == 0) { - result = url.substr(https_prefix.size()); - } else if (url.compare(0, http_prefix.size(), http_prefix) == 0) { - result = url.substr(http_prefix.size()); + if (url.compare(0, kHttpsPrefix.size(), kHttpsPrefix) == 0) { + result = url.substr(kHttpsPrefix.size()); + } else if (url.compare(0, kHttpPrefix.size(), kHttpPrefix) == 0) { + result = url.substr(kHttpPrefix.size()); } else { result = url; } @@ -37,6 +39,11 @@ std::string GetUrlContent(const std::string &url) { return result; } +bool IsHttpRequest(const std::string &url) { + return url.compare(0, kHttpPrefix.size(), kHttpPrefix) == 0 || + url.compare(0, kHttpsPrefix.size(), kHttpsPrefix) == 0; +} + } // namespace utils } // namespace api_manager } // namespace google diff --git a/contrib/endpoints/src/api_manager/utils/url_util.h b/contrib/endpoints/src/api_manager/utils/url_util.h index 2c002b37faf..cef8c2193d2 100644 --- a/contrib/endpoints/src/api_manager/utils/url_util.h +++ b/contrib/endpoints/src/api_manager/utils/url_util.h @@ -25,6 +25,8 @@ namespace utils { // processed string. std::string GetUrlContent(const std::string &url); +bool IsHttpRequest(const std::string &url); + } // namespace utils } // namespace api_manager } // namespace google From 9442a836988332f397f874e0b8ae9edafcd53425 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Mon, 10 Apr 2017 12:46:51 -0700 Subject: [PATCH 12/15] Merge from master to firebase (#237) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect (#38) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect * Fixed style. * Rebase Envoy (#41) * Update prototype to use iptables (#42) * Rebase to fixed Envoy (#43) * Handle HEAD request. (#34) * Handle HEAD request. * Try with GET if HEAD fails. * Address comments. * Format file. * Expose bazel target (#48) * Try again (#49) * Integrate with mixer client. (#55) * Integrate with mixer client. * Restore repositories.bzl back. * Add originIp and originHost attributes. (#56) * Add uuid-dev dependency in README.md (#45) * Extract originIp and OriginHost. (#57) * Extract originIp and OriginHost. * Make header x-forwarded-host const. * Update buckets for UI. (#58) * Update buckets for UI. * Only update time_distribution. * Add targetService attribute. (#59) * Use envoy new access_log handler for sending Report. (#60) * use access_log handler. * Not to use Loggable base class. * Update to the latest envoy with #396. (#61) * Fix tclap dependency fetching error (#62) * Integrate mixer client directly with envoy. (#66) * Integrate mixer client directly with envoy. * Send response header in Report. * rename filter name from esp to mixer. * add README. * Add release binary script. (#68) * Push tar.gz to GCS (#69) * Push tar.gz to GCS * Rename envoy_esp * Remove mixer_client from api_manager. (#72) * Update mixer client SHA. (#74) * Update readme. (#73) * Adds Jenkinsfile and updates release-binary to create a SHA. (#71) * Adds Jenkinsfile and update release-binary * Update Jenkinsfile and gitignore * Fixes typo and use normal build Node * Uses default bazel config * Using batch mode * Update bazel memory settings * Do not use Jenkins bazel env * Set .bazelrc for postsubmit * Update grpc and protobuf (#70) * protobuf v3.2.0 * grpc v1.1.1 * Align auth lib with grpc 1.1.1 * Add sourceService. (#78) * Add script to build docker image. (#77) * Add script to build docker image. * Add start_envoy for docker image. * Use official attribute names (#80) * Use official attribute names * fix format * Creates a KEY for mixer client dep. Updates release-binary (#79) * Updated mixer repo to use a key for commit * release-binary skip build if file exists. * Update src/envoy/mixer/README. (#82) * Fix src/envoy/mixer/README.md (#85) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Not to use api_key if its service is not actived. (#109) * Update envoy and add c-ares (#107) * Update envoy and add c-ares depedencies * Update release script with debug and normal binary * remove debug ls * formatting * Send StatusCode Attributes to Mixer. (#110) * Add send_attribute filter. (#115) * Add send_attribute filter. * Fix format * rename variable serialized_attributes_ * Address the comments. * Fail request if api_key is not valid. (#116) * Fail request if api_key is not valid. * Format code. * Update comments. * Address comment. * Rename response.http.code (#125) * Send headers as string map. (#129) * Send headers as string map. * Remove origin.ip and origin.host. * Fix format * unify bazel's docker build targets with other istio repos (#127) * update base debug docker image reference (#133) * Update postsubmit to create docker images (#132) * Adding config release for bazel build (#135) * Fix mixer client crash. (#136) * Get mixerclient with response parsing. (#138) * Update nghttp2 to sync with envoy (#140) * Fix src/envoy/mixer/README.md * Update nghttp2 to sync with envoy * update * fix typo * Populate origin.user attribute from the SAN field of client cert (#142) * Test * test * test * revert file * address comments * test * fix typo * fix format * fix format * Update to latest mixer_client. (#145) * Update to latest mixer_client. * Updated the sha. * Not call report if decodeHeaders is not called. (#150) * Update mixerclient with sync-ed grpc write and fail-fast. (#155) * Update mixerclient with sync-ed write and fail-fast. * Update to latest test. * Update again * Update envoy to PR553 (#156) * Update envoy to PR553 * Update libevent to 2.1.8 * Uses a specific version of the Shared Pipeline lib (#158) * Update lyft/envoy commit Id to latest. (#161) * Update lyft/envoy commit Id to latest. * Remove the comment about pull request * Add new line - will delete in next commit. * Update repositories.bzl (#169) * Always set response latency (#172) * Update mixerclient to sync_transport change. (#178) * Use opaque config to turn on/off forward attribute and mixer filter (#179) * Modify mixer filter * Swap defaults * Make the filter decoder only * cache mixer disabled decision * Fix a bug in opaque config change and test it out (#182) * Fix a bug and test it out * Update filter type * Update README.md * Update mixer client to mixer api with gogoproto. (#184) * Move .bazelrc to tools/bazel.rc (#186) * Move .bazelrc to tools/bazel.rc * Update Jenkinsfile with latest version of pipeline * Support apikey based traffic restriction (#189) * b/36368559 support apikey based traffic restriction * Fixed code formatting * Fix crash in unreachable/overloaded RDS (#190) * Add mixer client end to end integration test. (#177) * Add mixer client end to end integration test. * Split some repositories into a separate file. * use real mixer for fake mixer_server. * Test repository * use mixer bzl file. * Use mixer repositories * Not to use mixer repository. * Add return line at the end of WORKSPACE. * Fix broken link (#193) * Make quota call (#192) * hookup quota call * Make quota call. * Update indent. * Update envoy and update configs (#195) * Update envoy and update configs * Use gcc-4.9 for travis * Use bazel 0.4.5 * Fix SHA of lightstep-tracer-common * Enable check cache and refactory mixer config loading (#197) * Refactory the mixer config loading. * fix format * Add integration test. * updated README.md * s/send/sent/ * Split into separate tests. (#201) * Update README on how to enable check cache. (#204) * Update README on how to enable check cache. * Update the comment. * build: support Envoy native Bazel build. (#210) * build: support Envoy native Bazel build. This patch switches the Envoy build from src/envoy/repositories.bzl to using the upstream native build. See https://github.com/lyft/envoy/pull/663 for the corresponding changes on the Envoy side. * Use Envoy master with BUILD.wip rename merged. * Fix clang-format issues. * Fixes bazel.rc issues (#212) * Fixes bazel rc issues * Update Jenkins to latest pipeline version * Fix go build (#224) * Use TranscoderInputStream to reduce confusion around ByteCount() (#225) * Add TranscoderInputStream to reduce confusion * fix_format * Merge latest changes from rate_limiting to master (#221) * Point to googleapi in service control client. (#91) * Point to googleapi in service control client. * Use git repository for service-control-client. * Merge latest changes from master (#104) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Extract quota config from service config. (#101) * Add metric_cost in config. * Remove group rules. * Call loadQuotaConfig in config::create. * Update latest update from master branch (#106) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Added quota contoll without the service control client library (#93) * Added quota contoll without the service control client library * Applied code review * Applied code review * Resolve conflicts * Resolve conflicts * Fixed format error reported by script/check-style * Fixed a bug at Aggregated::GetAuthToken that causes Segmentation Fault * Changed usage of template funcion * Applied latest changes from the repo * Applied latest changes from the repo * Applied latest changes from the repo * Adde comments * Updated log information * Applied #101 * Changed metric_cost_map to metric_cost_vector * Fixed test case compilation error * Fixed test case compilation error * Add unit test for quota config. (#108) * Add unit test for quota config. * Add comments. * Update test specifics. * Merge latest changes from master branch (#112) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Not to use api_key if its service is not actived. (#109) * If QuotaControl service is not available, return utils::Status::OK (#113) * If QuotaControl service is not available, return utils::Status::OK * Updated comment * Return HTTP status code 429 on google.rpc.Code.RESOURCE_EXHAUSTED (#119) * Fixed incorrectly resolved conflicts (#123) * Added unit test cases for rate limiting (#124) * Fixed incorrectly resolved conflicts * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Rename response.http.code (#125) (#128) * Added handling of error code QUOTA_SYSTEM_UNAVAILABLE (#148) * Integrated service control client library with quota cache aggregation (#149) * Fixed error on merge (#151) * Integrated service control client library with quota cache aggregation * Fixed error on merge * Fixed the compatibility issue with the latest update on esp (#152) * Removed copied proto files (#208) * Set default allocate quota request timeout to 1sec and applied latest service control client library change (#211) * Merged key_restriction related changes from master (#213) * Merge latest changes from master branch (#217) * Not call report if decodeHeaders is not called. (#150) * Update mixerclient with sync-ed grpc write and fail-fast. (#155) * Update mixerclient with sync-ed write and fail-fast. * Update to latest test. * Update again * Update envoy to PR553 (#156) * Update envoy to PR553 * Update libevent to 2.1.8 * Uses a specific version of the Shared Pipeline lib (#158) * Update lyft/envoy commit Id to latest. (#161) * Update lyft/envoy commit Id to latest. * Remove the comment about pull request * Add new line - will delete in next commit. * Update repositories.bzl (#169) * Always set response latency (#172) * Update mixerclient to sync_transport change. (#178) * Use opaque config to turn on/off forward attribute and mixer filter (#179) * Modify mixer filter * Swap defaults * Make the filter decoder only * cache mixer disabled decision * Fix a bug in opaque config change and test it out (#182) * Fix a bug and test it out * Update filter type * Update README.md * Update mixer client to mixer api with gogoproto. (#184) * Move .bazelrc to tools/bazel.rc (#186) * Move .bazelrc to tools/bazel.rc * Update Jenkinsfile with latest version of pipeline * Support apikey based traffic restriction (#189) * b/36368559 support apikey based traffic restriction * Fixed code formatting * Fix crash in unreachable/overloaded RDS (#190) * Add mixer client end to end integration test. (#177) * Add mixer client end to end integration test. * Split some repositories into a separate file. * use real mixer for fake mixer_server. * Test repository * use mixer bzl file. * Use mixer repositories * Not to use mixer repository. * Add return line at the end of WORKSPACE. * Fix broken link (#193) * Make quota call (#192) * hookup quota call * Make quota call. * Update indent. * Update envoy and update configs (#195) * Update envoy and update configs * Use gcc-4.9 for travis * Use bazel 0.4.5 * Fix SHA of lightstep-tracer-common * Enable check cache and refactory mixer config loading (#197) * Refactory the mixer config loading. * fix format * Add integration test. * updated README.md * s/send/sent/ * Split into separate tests. (#201) * Update README on how to enable check cache. (#204) * Update README on how to enable check cache. * Update the comment. * build: support Envoy native Bazel build. (#210) * build: support Envoy native Bazel build. This patch switches the Envoy build from src/envoy/repositories.bzl to using the upstream native build. See https://github.com/lyft/envoy/pull/663 for the corresponding changes on the Envoy side. * Use Envoy master with BUILD.wip rename merged. * Fix clang-format issues. * Fixes bazel.rc issues (#212) * Fixes bazel rc issues * Update Jenkins to latest pipeline version * Updated the commit id of cloudendpoints/service-control-client-cxx (#218) * Update commitid of cloudendpoints/service-control-client-cxx repo (#220) * Send delta metrics for intermediate reports. (#219) * Send delta metrics for intermediate reports. * Move last_request_bytes/last_response_bytes to RequestContext. * Handle final report. * Address comment. * Update attributes to match the canonical attribute list. (#232) * Update response.http.code to response.code and response.latency to response.duration to line up with the canonical attributes in istio/istio.github.io/docs/concepts/attributes.md * Format according to clang-format * Add envoy Buffer based TranscoderInputStream (#231) * Add envoy Buffer based TranscoderInputStream * fix format * A few doc changes for consistency across repos. (#235) * Add repositories.bzl * Added missing export setting in bazel configuration (#236) * Added export missing in bazel configuration * Added export missing in bazel configuration --- .travis.yml | 30 +- BUILD | 4 + CONTRIBUTING.md | 40 +-- Jenkinsfile | 29 +- README.md | 13 +- WORKSPACE | 16 ++ .../endpoints/include/api_manager/method.h | 4 + contrib/endpoints/repositories.bzl | 21 +- contrib/endpoints/src/api_manager/BUILD | 2 + .../api_manager/auth/service_account_token.h | 1 + .../src/api_manager/check_workflow.cc | 3 + contrib/endpoints/src/api_manager/config.cc | 20 ++ contrib/endpoints/src/api_manager/config.h | 3 + .../endpoints/src/api_manager/config_test.cc | 85 ++++++ .../api_manager/context/request_context.cc | 46 ++- .../src/api_manager/context/request_context.h | 7 + .../endpoints/src/api_manager/method_impl.h | 18 +- .../src/api_manager/mock_method_info.h | 2 + .../src/api_manager/proto/server_config.proto | 18 ++ .../src/api_manager/quota_control.cc | 57 ++++ .../endpoints/src/api_manager/quota_control.h | 33 +++ .../src/api_manager/service_control/BUILD | 13 + .../api_manager/service_control/aggregated.cc | 200 ++++++++++--- .../api_manager/service_control/aggregated.h | 21 +- .../service_control/aggregated_test.cc | 131 +++++++++ .../allocate_quota_response_test.cc | 184 ++++++++++++ .../src/api_manager/service_control/info.h | 14 +- .../api_manager/service_control/interface.h | 11 + .../src/api_manager/service_control/proto.cc | 174 +++++++++++ .../src/api_manager/service_control/proto.h | 9 + .../api_manager/service_control/proto_test.cc | 68 +++++ .../testdata/allocate_quota_request.golden | 36 +++ .../allocate_quota_request_android_ios.golden | 48 ++++ ...locate_quota_request_no_method_name.golden | 35 +++ .../testdata/check_request_android_ios.golden | 43 +++ .../src/api_manager/service_control/url.cc | 2 + .../src/api_manager/service_control/url.h | 2 + .../api_manager/service_control/url_test.cc | 4 + contrib/endpoints/src/grpc/transcoding/BUILD | 14 + .../src/grpc/transcoding/message_reader.cc | 10 +- .../src/grpc/transcoding/message_reader.h | 11 +- .../src/grpc/transcoding/message_stream.cc | 30 +- .../src/grpc/transcoding/message_stream.h | 4 +- .../grpc/transcoding/message_stream_test.cc | 58 ++-- .../response_to_json_translator.cc | 3 +- .../transcoding/response_to_json_translator.h | 3 +- .../src/grpc/transcoding/test_common.cc | 2 +- .../src/grpc/transcoding/test_common.h | 9 +- .../src/grpc/transcoding/transcoder.h | 18 +- .../grpc/transcoding/transcoder_factory.cc | 20 +- .../src/grpc/transcoding/transcoder_factory.h | 3 +- .../transcoding/transcoder_input_stream.h | 35 +++ .../src/grpc/transcoding/transcoder_test.cc | 7 +- src/envoy/BUILD | 34 --- src/envoy/mixer/BUILD | 24 +- src/envoy/mixer/README.md | 96 +++++-- src/envoy/mixer/config.cc | 97 +++++++ src/envoy/mixer/config.h | 54 ++++ src/envoy/mixer/envoy.conf.template | 30 +- src/envoy/mixer/http_control.cc | 66 +++-- src/envoy/mixer/http_control.h | 10 +- src/envoy/mixer/http_filter.cc | 125 +++++--- src/envoy/mixer/integration_test/BUILD | 64 +++++ .../mixer/integration_test/attributes.go | 128 +++++++++ .../integration_test/check_cache_test.go | 40 +++ .../integration_test/check_report_test.go | 154 ++++++++++ src/envoy/mixer/integration_test/envoy.go | 88 ++++++ .../mixer/integration_test/envoy_conf.go | 244 ++++++++++++++++ .../integration_test/failed_request_test.go | 161 +++++++++++ .../mixer/integration_test/http_client.go | 80 ++++++ .../mixer/integration_test/http_server.go | 111 +++++++ .../mixer/integration_test/mixer_server.go | 127 ++++++++ .../mixer/integration_test/quota_test.go | 64 +++++ .../mixer/integration_test/repositories.bzl | 272 ++++++++++++++++++ src/envoy/mixer/integration_test/setup.go | 93 ++++++ .../mixer/integration_test/test_suite.bzl | 29 ++ src/envoy/mixer/repositories.bzl | 2 +- src/envoy/mixer/utils.cc | 14 - src/envoy/mixer/utils.h | 3 - src/envoy/repositories.bzl | 238 ++------------- src/envoy/transcoding/BUILD | 40 +++ src/envoy/transcoding/envoy_input_stream.cc | 63 ++++ src/envoy/transcoding/envoy_input_stream.h | 51 ++++ .../transcoding/envoy_input_stream_test.cc | 97 +++++++ .bazelrc => tools/bazel.rc | 0 tools/bazel.rc.jenkins | 8 + .bazelrc.travis => tools/bazel.rc.travis | 0 87 files changed, 3788 insertions(+), 593 deletions(-) create mode 100644 contrib/endpoints/src/api_manager/quota_control.cc create mode 100644 contrib/endpoints/src/api_manager/quota_control.h create mode 100644 contrib/endpoints/src/api_manager/service_control/allocate_quota_response_test.cc create mode 100644 contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request.golden create mode 100644 contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_android_ios.golden create mode 100644 contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_no_method_name.golden create mode 100644 contrib/endpoints/src/api_manager/service_control/testdata/check_request_android_ios.golden create mode 100644 contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h create mode 100644 src/envoy/mixer/config.cc create mode 100644 src/envoy/mixer/config.h create mode 100644 src/envoy/mixer/integration_test/BUILD create mode 100644 src/envoy/mixer/integration_test/attributes.go create mode 100644 src/envoy/mixer/integration_test/check_cache_test.go create mode 100644 src/envoy/mixer/integration_test/check_report_test.go create mode 100644 src/envoy/mixer/integration_test/envoy.go create mode 100644 src/envoy/mixer/integration_test/envoy_conf.go create mode 100644 src/envoy/mixer/integration_test/failed_request_test.go create mode 100644 src/envoy/mixer/integration_test/http_client.go create mode 100644 src/envoy/mixer/integration_test/http_server.go create mode 100644 src/envoy/mixer/integration_test/mixer_server.go create mode 100644 src/envoy/mixer/integration_test/quota_test.go create mode 100644 src/envoy/mixer/integration_test/repositories.bzl create mode 100644 src/envoy/mixer/integration_test/setup.go create mode 100644 src/envoy/mixer/integration_test/test_suite.bzl create mode 100644 src/envoy/transcoding/BUILD create mode 100644 src/envoy/transcoding/envoy_input_stream.cc create mode 100644 src/envoy/transcoding/envoy_input_stream.h create mode 100644 src/envoy/transcoding/envoy_input_stream_test.cc rename .bazelrc => tools/bazel.rc (100%) create mode 100644 tools/bazel.rc.jenkins rename .bazelrc.travis => tools/bazel.rc.travis (100%) diff --git a/.travis.yml b/.travis.yml index 88be42c285b..845ac9bbd8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,27 @@ sudo: required -dist: xenial + +dist: trusty + +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.9 + - g++-4.9 + - wget branches: except: - stable -lang: go - -go: - - 1.7.x +language: cpp jdk: - oraclejdk8 env: - - BAZEL_VERSION=0.4.3 - -addons: - apt: - packages: - - wget + - BAZEL_VERSION=0.4.5 cache: directories: @@ -35,12 +37,12 @@ before_install: - sudo dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb - sudo apt-get -f install -qqy uuid-dev - cd ${TRAVIS_BUILD_DIR} - - mv .bazelrc .bazelrc.orig - - cat .bazelrc.travis .bazelrc.orig > .bazelrc + - mv tools/bazel.rc tools/bazel.rc.orig + - cat tools/bazel.rc.travis tools/bazel.rc.orig > tools/bazel.rc script: - script/check-style - - bazel --output_base=${HOME}/bazel/outbase test //... + - CC=/usr/bin/gcc-4.9 CXX=/usr/bin/g++-4.9 bazel --output_base=${HOME}/bazel/outbase test //... notifications: slack: istio-dev:wEEEbaabdP5ieCgDOFetA9nX diff --git a/BUILD b/BUILD index ac06d245289..510620e8246 100644 --- a/BUILD +++ b/BUILD @@ -24,3 +24,7 @@ config_setting( }, visibility = ["//visibility:public"], ) + +load("@io_bazel_rules_go//go:def.bzl", "go_prefix") + +go_prefix("istio.io/proxy") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 936e3c8f91b..1c21f8a4311 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,37 +1,5 @@ -# Contributing guidelines +# Contribution guidelines -So, you want to hack on the Istio Proxy? Yay! - -- [Contributor license agreements](#contributor-license-agreements) -- [Contributing a patch](#contributing-a-patch) - -## Contributor license agreements - -We'd love to accept your patches! Before we can take them, you have to jump a -few legal hurdles. - -Please fill out the [Google CLA](https://cla.developers.google.com). - -Once you are CLA'ed, we'll be able to accept your pull requests. This is necessary -because you own the copyright to your changes, even after your contribution -becomes part of this project. So this agreement simply gives us permission -to use and redistribute your contributions as part of the project. - -***NOTE***: Only original source code from you and other people that have -signed the CLA can be accepted into the repository. This policy does not -apply to [third_party](third_party/) and [vendor](vendor/). - -## Contributing a patch - -If you're working on an existing issue, simply respond to the issue and express -interest in working on it. This helps other people know that the issue is -active, and hopefully prevents duplicated efforts. - -If you want to work on a new idea of relatively small scope: - -1. Submit an issue describing your proposed change to the repo in question. -1. The repo owners will respond to your issue promptly. -1. If your proposed change is accepted, and you haven't already done so, sign a - Contributor License Agreement (see details above). -1. Fork the repo, develop, and test your changes. -1. Submit a pull request. +So, you want to hack on the Istio proxy? Yay! Please refer to Istio's overall +[contribution guidelines](https://github.com/istio/istio/blob/master/CONTRIBUTING.md) +to find out how you can help. diff --git a/Jenkinsfile b/Jenkinsfile index 9c59a146e50..c1aa89e3f25 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!groovy -@Library('testutils') +@Library('testutils@stable-3e4d089') import org.istio.testutils.Utilities import org.istio.testutils.GitUtilities @@ -12,20 +12,17 @@ def utils = new Utilities() def bazel = new Bazel() mainFlow(utils) { - pullRequest(utils) { - node { - gitUtils.initialize() - // Proxy does build work correctly with Hazelcast. - // Must use .bazelrc.jenkins - bazel.setVars('', '') - } - - if (utils.runStage('PRESUBMIT')) { - presubmit(gitUtils, bazel) - } - if (utils.runStage('POSTSUBMIT')) { - postsubmit(gitUtils, bazel, utils) - } + node { + gitUtils.initialize() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') + } + if (utils.runStage('PRESUBMIT')) { + presubmit(gitUtils, bazel) + } + if (utils.runStage('POSTSUBMIT')) { + postsubmit(gitUtils, bazel, utils) } } @@ -62,4 +59,4 @@ def postsubmit(gitUtils, bazel, utils) { utils.publishDockerImages(images, tags, 'release') } } -} \ No newline at end of file +} diff --git a/README.md b/README.md index 26a33ca4b89..9c478e2f3f0 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,7 @@ Server Side Features: - *Monitoring & Logging*. The Proxy can report server-side metrics and logs to the Mixer. -To learn more... - -- User guide coming soon! -- [Contributing to the project](./CONTRIBUTING.md) - -### Filing issues - -If you have a question about an Istio proxy or have a problem using one, please -[file an issue](https://github.com/istio/proxy/issues/new). +Please see [istio.io](https://istio.io) +to learn about the overall Istio project and how to get in touch with us. To learn how you can +contribute to any of the Istio components, including the proxy, please +see the Istio [contribution guidelines](https://github.com/istio/istio/blob/master/CONTRIBUTING.md). diff --git a/WORKSPACE b/WORKSPACE index ddfa05d578b..6f387192fb9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -96,3 +96,19 @@ http_file( url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug-" + DEBUG_BASE_IMAGE_SHA + ".tar.gz", sha256 = DEBUG_BASE_IMAGE_SHA, ) + +# Following go repositories are for building go integration test for mixer filter. +git_repository( + name = "io_bazel_rules_go", + commit = "2d9f328a9723baf2d037ba9db28d9d0e30683938", # Apr 6, 2017 (buildifier fix) + remote = "https://github.com/bazelbuild/rules_go.git", +) + +git_repository( + name = "org_pubref_rules_protobuf", + commit = "d42e895387c658eda90276aea018056fcdcb30e4", # Mar 07 2017 (gogo* support) + remote = "https://github.com/pubref/rules_protobuf", +) + +load("//src/envoy/mixer/integration_test:repositories.bzl", "go_mixer_repositories") +go_mixer_repositories() diff --git a/contrib/endpoints/include/api_manager/method.h b/contrib/endpoints/include/api_manager/method.h index 6b7c01072b8..1680b6950ff 100644 --- a/contrib/endpoints/include/api_manager/method.h +++ b/contrib/endpoints/include/api_manager/method.h @@ -89,6 +89,10 @@ class MethodInfo { // Get the names of url system parameters virtual const std::set &system_query_parameter_names() const = 0; + + // Get quota metric cost vector + virtual const std::vector> &metric_cost_vector() + const = 0; }; } // namespace api_manager diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index c77a3f72f58..0d66babfcc8 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -214,7 +214,7 @@ def googleapis_repositories(protobuf_repo="@protobuf_git//", bind=True): licenses(["notice"]) -load("{}:protobuf.bzl", "cc_proto_library") +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") exports_files(glob(["google/**"])) @@ -254,16 +254,18 @@ cc_proto_library( "google/api/control.proto", "google/api/documentation.proto", "google/api/endpoint.proto", - "google/api/experimental/authorization_config.proto", - "google/api/experimental/experimental.proto", "google/api/http.proto", "google/api/label.proto", "google/api/log.proto", "google/api/logging.proto", "google/api/metric.proto", + "google/api/experimental/experimental.proto", + "google/api/experimental/authorization_config.proto", "google/api/monitored_resource.proto", "google/api/monitoring.proto", + "google/api/quota.proto", "google/api/service.proto", + "google/api/source_info.proto", "google/api/system_parameter.proto", "google/api/usage.proto", ], @@ -292,10 +294,9 @@ cc_proto_library( ) """.format(protobuf_repo) - native.new_git_repository( name = "googleapis_git", - commit = "412867fb105722fb9d2cd9af90af1f8f120de238", + commit = "2fe0050bd2a6d4c6ba798c0311f0b149b8997314", remote = "https://github.com/googleapis/googleapis.git", build_file_content = BUILD, ) @@ -326,7 +327,7 @@ def servicecontrol_client_repositories(bind=True): native.git_repository( name = "servicecontrol_client_git", - commit = "d739d755365c6a13d0b4164506fd593f53932f5d", + commit = "3d1a30d9221e700542eeaaf20eab69faddb63894", remote = "https://github.com/cloudendpoints/service-control-client-cxx.git", ) @@ -335,3 +336,11 @@ def servicecontrol_client_repositories(bind=True): name = "servicecontrol_client", actual = "@servicecontrol_client_git//:service_control_client_lib", ) + native.bind( + name = "quotacontrol", + actual = "@servicecontrol_client_git//proto:quotacontrol", + ) + native.bind( + name = "quotacontrol_genproto", + actual = "@servicecontrol_client_git//proto:quotacontrol_genproto", + ) diff --git a/contrib/endpoints/src/api_manager/BUILD b/contrib/endpoints/src/api_manager/BUILD index 83351e10a55..ef95d32885e 100644 --- a/contrib/endpoints/src/api_manager/BUILD +++ b/contrib/endpoints/src/api_manager/BUILD @@ -96,6 +96,8 @@ cc_library( "method_impl.cc", "path_matcher.cc", "path_matcher_node.cc", + "quota_control.cc", + "quota_control.h", "request_handler.cc", ], linkopts = select({ diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token.h b/contrib/endpoints/src/api_manager/auth/service_account_token.h index 3a3453daf91..ef3cabce600 100644 --- a/contrib/endpoints/src/api_manager/auth/service_account_token.h +++ b/contrib/endpoints/src/api_manager/auth/service_account_token.h @@ -68,6 +68,7 @@ class ServiceAccountToken { // JWT token for accessing the http endpoints defined in Firebase Rules. JWT_TOKEN_FOR_AUTHORIZATION_SERVICE, + JWT_TOKEN_FOR_QUOTA_CONTROL, JWT_TOKEN_TYPE_MAX, }; // Set audience. Only calcualtes JWT token with specified audience. diff --git a/contrib/endpoints/src/api_manager/check_workflow.cc b/contrib/endpoints/src/api_manager/check_workflow.cc index 7c869ab30cc..79cd74953db 100644 --- a/contrib/endpoints/src/api_manager/check_workflow.cc +++ b/contrib/endpoints/src/api_manager/check_workflow.cc @@ -19,6 +19,7 @@ #include "contrib/endpoints/src/api_manager/check_security_rules.h" #include "contrib/endpoints/src/api_manager/check_service_control.h" #include "contrib/endpoints/src/api_manager/fetch_metadata.h" +#include "contrib/endpoints/src/api_manager/quota_control.h" using ::google::api_manager::utils::Status; @@ -36,6 +37,8 @@ void CheckWorkflow::RegisterAll() { Register(CheckServiceControl); // Check Security Rules. Register(CheckSecurityRules); + // Quota control + Register(QuotaControl); } void CheckWorkflow::Register(CheckHandler handler) { diff --git a/contrib/endpoints/src/api_manager/config.cc b/contrib/endpoints/src/api_manager/config.cc index 66ac9aff1c1..a548d1450b5 100644 --- a/contrib/endpoints/src/api_manager/config.cc +++ b/contrib/endpoints/src/api_manager/config.cc @@ -113,6 +113,23 @@ MethodInfoImpl *Config::GetOrCreateMethodInfoImpl(const string &name, return i->second.get(); } +bool Config::LoadQuotaRule(ApiManagerEnvInterface *env) { + for (const auto &rule : service_.quota().metric_rules()) { + auto method = utils::FindOrNull(method_map_, rule.selector()); + if (method) { + for (auto &metric_cost : rule.metric_costs()) { + (*method)->add_metric_cost(metric_cost.first, metric_cost.second); + } + } else { + env->LogError("Metric rule with selector " + rule.selector() + + "is mismatched."); + return false; + } + } + + return true; +} + bool Config::LoadHttpMethods(ApiManagerEnvInterface *env, PathMatcherBuilder *pmb) { std::set all_urls, urls_with_options; @@ -443,6 +460,9 @@ std::unique_ptr Config::Create(ApiManagerEnvInterface *env, if (!config->LoadBackends(env)) { return nullptr; } + if (!config->LoadQuotaRule(env)) { + return nullptr; + } return config; } diff --git a/contrib/endpoints/src/api_manager/config.h b/contrib/endpoints/src/api_manager/config.h index f7cca3838ff..b5d56d58461 100644 --- a/contrib/endpoints/src/api_manager/config.h +++ b/contrib/endpoints/src/api_manager/config.h @@ -25,6 +25,7 @@ #include "contrib/endpoints/src/api_manager/method_impl.h" #include "contrib/endpoints/src/api_manager/path_matcher.h" #include "contrib/endpoints/src/api_manager/proto/server_config.pb.h" +#include "google/api/quota.pb.h" #include "google/api/service.pb.h" namespace google { @@ -113,6 +114,8 @@ class Config { // Load SystemParameters info to MethodInfo. bool LoadSystemParameters(ApiManagerEnvInterface *env); + bool LoadQuotaRule(ApiManagerEnvInterface *env); + // Gets the MethodInfoImpl creating it if necessary MethodInfoImpl *GetOrCreateMethodInfoImpl(const std::string &name, const std::string &api_name, diff --git a/contrib/endpoints/src/api_manager/config_test.cc b/contrib/endpoints/src/api_manager/config_test.cc index 2302bd5cbf7..98020a79811 100644 --- a/contrib/endpoints/src/api_manager/config_test.cc +++ b/contrib/endpoints/src/api_manager/config_test.cc @@ -941,6 +941,91 @@ TEST(Config, TestFirebaseServerCheckWithoutServiceConfigWithServerConfig) { ASSERT_EQ(config->GetFirebaseServer(), "https://myfirebaseserver.com/"); } + +TEST(Config, TestInvalidMetricRules) { + MockApiManagerEnvironmentWithLog env; + // There is no http.rule or api.method to match the selector. + static const char config_text[] = R"( +name: "Service.Name" +quota { + metric_rules { + selector: "GetShelves" + metric_costs { + key: "test.googleapis.com/operation/read_book" + value: 100 + } + } +} +)"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + EXPECT_EQ(nullptr, config); +} + +TEST(Config, TestMetricRules) { + MockApiManagerEnvironmentWithLog env; + static const char config_text[] = R"( +name: "Service.Name" +http { + rules { + selector: "DeleteShelf" + delete: "/shelves" + } + rules { + selector: "GetShelves" + get: "/shelves" + } +} +quota { + metric_rules { + selector: "GetShelves" + metric_costs { + key: "test.googleapis.com/operation/get_shelves" + value: 100 + } + metric_costs { + key: "test.googleapis.com/operation/request" + value: 10 + } + } + metric_rules { + selector: "DeleteShelf" + metric_costs { + key: "test.googleapis.com/operation/delete_shelves" + value: 200 + } + } +} +)"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + ASSERT_TRUE(config); + + const MethodInfo *method1 = config->GetMethodInfo("GET", "/shelves"); + ASSERT_NE(nullptr, method1); + + std::vector> metric_cost_vector = + method1->metric_cost_vector(); + std::sort(metric_cost_vector.begin(), metric_cost_vector.end()); + ASSERT_EQ(2, metric_cost_vector.size()); + ASSERT_EQ("test.googleapis.com/operation/get_shelves", + metric_cost_vector[0].first); + ASSERT_EQ(100, metric_cost_vector[0].second); + + ASSERT_EQ("test.googleapis.com/operation/request", + metric_cost_vector[1].first); + ASSERT_EQ(10, metric_cost_vector[1].second); + + const MethodInfo *method2 = config->GetMethodInfo("DELETE", "/shelves"); + ASSERT_NE(nullptr, method1); + + metric_cost_vector = method2->metric_cost_vector(); + ASSERT_EQ(1, metric_cost_vector.size()); + ASSERT_EQ("test.googleapis.com/operation/delete_shelves", + metric_cost_vector[0].first); + ASSERT_EQ(200, metric_cost_vector[0].second); +} + } // namespace } // namespace api_manager diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index 75bcb177f9d..4703e2e5ee7 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -51,6 +51,16 @@ const char kDefaultApiKeyQueryName1[] = "key"; const char kDefaultApiKeyQueryName2[] = "api_key"; const char kDefaultApiKeyHeaderName[] = "x-api-key"; +// Header for android package name, used for api key restriction check. +const char kXAndroidPackage[] = "x-android-package"; + +// Header for android certificate fingerprint, used for api key restriction +// check. +const char kXAndroidCert[] = "x-android-cert"; + +// Header for IOS bundle identifier, used for api key restriction check. +const char kXIosBundleId[] = "x-ios-bundle-identifier"; + // Default location const char kDefaultLocation[] = "us-central1"; @@ -71,7 +81,9 @@ RequestContext::RequestContext(std::shared_ptr service_context, std::unique_ptr request) : service_context_(service_context), request_(std::move(request)), - is_first_report_(true) { + is_first_report_(true), + last_request_bytes_(0), + last_response_bytes_(0) { start_time_ = std::chrono::system_clock::now(); last_report_time_ = std::chrono::steady_clock::now(); operation_id_ = GenerateUUID(); @@ -225,6 +237,20 @@ void RequestContext::FillCheckRequestInfo( service_control::CheckRequestInfo *info) { FillOperationInfo(info); info->allow_unregistered_calls = method()->allow_unregistered_calls(); + + request_->FindHeader(kXAndroidPackage, &info->android_package_name); + request_->FindHeader(kXAndroidCert, &info->android_cert_fingerprint); + request_->FindHeader(kXIosBundleId, &info->ios_bundle_id); +} + +void RequestContext::FillAllocateQuotaRequestInfo( + service_control::QuotaRequestInfo *info) { + FillOperationInfo(info); + + info->client_ip = request_->GetClientIP(); + info->method_name = this->method_call_.method_info->name(); + info->metric_cost_vector = + &this->method_call_.method_info->metric_cost_vector(); } void RequestContext::FillReportRequestInfo( @@ -243,13 +269,23 @@ void RequestContext::FillReportRequestInfo( info->auth_audience = auth_audience_; if (!info->is_final_report) { - info->request_bytes = request_->GetGrpcRequestBytes(); - info->response_bytes = request_->GetGrpcResponseBytes(); + // Make sure we send delta metrics for intermediate reports. + info->request_bytes = request_->GetGrpcRequestBytes() - last_request_bytes_; + info->response_bytes = + request_->GetGrpcResponseBytes() - last_response_bytes_; + last_request_bytes_ += info->request_bytes; + last_response_bytes_ += info->response_bytes; } else { info->request_size = response->GetRequestSize(); info->response_size = response->GetResponseSize(); - info->request_bytes = info->request_size; - info->response_bytes = info->response_size; + info->request_bytes = info->request_size - last_request_bytes_; + if (info->request_bytes < 0) { + info->request_bytes = 0; + } + info->response_bytes = info->response_size - last_response_bytes_; + if (info->response_bytes < 0) { + info->response_bytes = 0; + } info->streaming_request_message_counts = request_->GetGrpcRequestMessageCounts(); diff --git a/contrib/endpoints/src/api_manager/context/request_context.h b/contrib/endpoints/src/api_manager/context/request_context.h index 57706c27a2e..3fdb15526b8 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.h +++ b/contrib/endpoints/src/api_manager/context/request_context.h @@ -68,6 +68,9 @@ class RequestContext { // Fill CheckRequestInfo void FillCheckRequestInfo(service_control::CheckRequestInfo *info); + // FillAllocateQuotaRequestInfo + void FillAllocateQuotaRequestInfo(service_control::QuotaRequestInfo *info); + // Fill ReportRequestInfo void FillReportRequestInfo(Response *response, service_control::ReportRequestInfo *info); @@ -186,6 +189,10 @@ class RequestContext { // The time point of last intermediate report std::chrono::steady_clock::time_point last_report_time_; + + // The accumulated data sent till last intermediate report + int64_t last_request_bytes_; + int64_t last_response_bytes_; }; } // namespace context diff --git a/contrib/endpoints/src/api_manager/method_impl.h b/contrib/endpoints/src/api_manager/method_impl.h index e5739d639bc..d6eaba33724 100644 --- a/contrib/endpoints/src/api_manager/method_impl.h +++ b/contrib/endpoints/src/api_manager/method_impl.h @@ -18,6 +18,7 @@ #include #include #include +#include #include "contrib/endpoints/include/api_manager/method.h" #include "contrib/endpoints/src/api_manager/utils/stl_util.h" @@ -62,6 +63,10 @@ class MethodInfoImpl : public MethodInfo { const std::string &backend_address() const { return backend_address_; } + const std::vector> &metric_cost_vector() const { + return metric_cost_vector_; + } + const std::string &rpc_method_full_name() const { return rpc_method_full_name_; } @@ -90,6 +95,10 @@ class MethodInfoImpl : public MethodInfo { url_query_parameters_[name].push_back(url_query_parameter); } + void add_metric_cost(const std::string &metric, int64_t cost) { + metric_cost_vector_.push_back(std::make_pair(metric, cost)); + } + // After add all system parameters, lookup some of them to cache // their lookup results. void process_system_parameters(); @@ -139,13 +148,13 @@ class MethodInfoImpl : public MethodInfo { // such as API Key)? bool allow_unregistered_calls_; // Issuers to allowed audiences map. - std::map > issuer_audiences_map_; + std::map> issuer_audiences_map_; // system parameter map of parameter name to http_header name. - std::map > http_header_parameters_; + std::map> http_header_parameters_; // system parameter map of parameter name to url query parameter name. - std::map > url_query_parameters_; + std::map> url_query_parameters_; // all the names of system query parameters std::set system_query_parameter_names_; @@ -175,6 +184,9 @@ class MethodInfoImpl : public MethodInfo { // Whether the response is streaming or not. bool response_streaming_; + + // map of metric and its cost + std::vector> metric_cost_vector_; }; typedef std::unique_ptr MethodInfoImplPtr; diff --git a/contrib/endpoints/src/api_manager/mock_method_info.h b/contrib/endpoints/src/api_manager/mock_method_info.h index a7de28e3422..6b78e86228b 100644 --- a/contrib/endpoints/src/api_manager/mock_method_info.h +++ b/contrib/endpoints/src/api_manager/mock_method_info.h @@ -48,6 +48,8 @@ class MockMethodInfo : public MethodInfo { MOCK_CONST_METHOD0(response_streaming, bool()); MOCK_CONST_METHOD0(system_query_parameter_names, const std::set&()); + MOCK_CONST_METHOD0(metric_cost_vector, + const std::vector>&()); }; } // namespace api_manager diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index d3f3189fcf3..5e26613f140 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -69,6 +69,13 @@ message ServiceControlConfig { // The intermediate reports for streaming calls should not be more frequent // than this value (in seconds) int32 intermediate_report_min_interval = 7; + + // Quota aggregator config + QuotaAggregatorConfig quota_aggregator_config = 8; + + // Timeout in milliseconds on service control allocate quota requests. + // If the value is <= 0, default timeout is 5000 milliseconds. + int32 quota_timeout_ms = 9; } // Check aggregator config @@ -85,6 +92,17 @@ message CheckAggregatorConfig { int32 response_expiration_ms = 3; } +// Quota aggregator config +message QuotaAggregatorConfig { + // The maximum number of cache entries that can be kept in the aggregation + // cache. Cache is disabled when entries <= 0. + int32 cache_entries = 1; + + // The maximum milliseconds before aggregated quota requests are refreshed to + // the server. + int32 refresh_interval_ms = 2; +} + // Report aggregator config message ReportAggregatorConfig { // The maximum number of cache entries that can be kept in the aggregation diff --git a/contrib/endpoints/src/api_manager/quota_control.cc b/contrib/endpoints/src/api_manager/quota_control.cc new file mode 100644 index 00000000000..8a5b65322d6 --- /dev/null +++ b/contrib/endpoints/src/api_manager/quota_control.cc @@ -0,0 +1,57 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include + +#include "contrib/endpoints/src/api_manager/cloud_trace/cloud_trace.h" +#include "contrib/endpoints/src/api_manager/quota_control.h" +#include "google/protobuf/stubs/status.h" + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { + +void QuotaControl(std::shared_ptr context, + std::function continuation) { + std::shared_ptr trace_span( + CreateSpan(context->cloud_trace(), "QuotaControl")); + + if (context->method()->metric_cost_vector().size() == 0) { + TRACE(trace_span) << "Quota control check is not needed"; + continuation(Status::OK); + return; + } + + service_control::QuotaRequestInfo info; + context->FillAllocateQuotaRequestInfo(&info); + context->service_context()->service_control()->Quota( + info, trace_span.get(), + [context, continuation, trace_span](utils::Status status) { + + TRACE(trace_span) << "Quota service control request returned with " + << "status " << status.ToString(); + + // quota control is using "failed open" policy. If the server is not + // available, allow the request to go. + continuation((status.code() == Code::UNAVAILABLE) ? utils::Status::OK + : status); + }); +} + +} // namespace service_control_client +} // namespace google diff --git a/contrib/endpoints/src/api_manager/quota_control.h b/contrib/endpoints/src/api_manager/quota_control.h new file mode 100644 index 00000000000..e4f94d6ac93 --- /dev/null +++ b/contrib/endpoints/src/api_manager/quota_control.h @@ -0,0 +1,33 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#ifndef API_MANAGER_QUOTA_CONTROL_H_ +#define API_MANAGER_QUOTA_CONTROL_H_ + +#include "contrib/endpoints/include/api_manager/utils/status.h" +#include "contrib/endpoints/src/api_manager/context/request_context.h" + +namespace google { +namespace api_manager { + +// Call service control quota. +void QuotaControl(std::shared_ptr, + std::function); + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_QUOTA_CONTROL_H_ diff --git a/contrib/endpoints/src/api_manager/service_control/BUILD b/contrib/endpoints/src/api_manager/service_control/BUILD index 73b722b4808..c0e35adca07 100644 --- a/contrib/endpoints/src/api_manager/service_control/BUILD +++ b/contrib/endpoints/src/api_manager/service_control/BUILD @@ -122,3 +122,16 @@ cc_test( "//external:googletest_main", ], ) + +cc_test( + name = "allocate_quota_response_test", + size = "small", + srcs = [ + "allocate_quota_response_test.cc", + ], + linkstatic = 1, + deps = [ + ":service_control", + "//external:googletest_main", + ], +) diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.cc b/contrib/endpoints/src/api_manager/service_control/aggregated.cc index e59a69dc8cc..f1ef15e7ee2 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated.cc +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.cc @@ -22,6 +22,8 @@ using ::google::api::servicecontrol::v1::CheckRequest; using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::AllocateQuotaRequest; +using ::google::api::servicecontrol::v1::AllocateQuotaResponse; using ::google::api::servicecontrol::v1::ReportRequest; using ::google::api::servicecontrol::v1::ReportResponse; using ::google::api_manager::proto::ServerConfig; @@ -29,6 +31,7 @@ using ::google::api_manager::utils::Status; using ::google::protobuf::util::error::Code; using ::google::service_control_client::CheckAggregationOptions; +using ::google::service_control_client::QuotaAggregationOptions; using ::google::service_control_client::ReportAggregationOptions; using ::google::service_control_client::ServiceControlClient; using ::google::service_control_client::ServiceControlClientOptions; @@ -40,6 +43,9 @@ namespace service_control { namespace { +const int kQuotaAggregationEntries = 10000; +const int kQuotaAggregationRefreshMs = 1000; + // Default config for check aggregator const int kCheckAggregationEntries = 10000; // Check doesn't support quota yet. It is safe to increase @@ -54,6 +60,8 @@ const int kReportAggregationFlushIntervalMs = 1000; // The default connection timeout for check requests. const int kCheckDefaultTimeoutInMs = 5000; +// The default connection timeout for allocate quota requests. +const int kAllocateQuotaDefaultTimeoutInMs = 1000; // The default connection timeout for report requests. const int kReportDefaultTimeoutInMs = 15000; @@ -69,6 +77,10 @@ const char application_proto[] = "application/x-protobuf"; const char servicecontrol_service[] = "/google.api.servicecontrol.v1.ServiceController"; +// The quota_control service name. used for as audience to generate JWT token. +const char quotacontrol_service[] = + "/google.api.servicecontrol.v1.QuotaController"; + // Generates CheckAggregationOptions. CheckAggregationOptions GetCheckAggregationOptions( const ServerConfig* server_config) { @@ -85,6 +97,24 @@ CheckAggregationOptions GetCheckAggregationOptions( kCheckAggregationExpirationMs); } +// Generate QuotaAggregationOptions +QuotaAggregationOptions GetQuotaAggregationOptions( + const ServerConfig* server_config) { + QuotaAggregationOptions option = QuotaAggregationOptions( + kQuotaAggregationEntries, kQuotaAggregationRefreshMs); + + if (server_config && server_config->has_service_control_config() && + server_config->service_control_config().has_quota_aggregator_config()) { + const auto& quota_config = + server_config->service_control_config().quota_aggregator_config(); + + option.num_entries = quota_config.cache_entries(); + option.refresh_interval_ms = quota_config.refresh_interval_ms(); + } + + return option; +} + // Generates ReportAggregationOptions. ReportAggregationOptions GetReportAggregationOptions( const ServerConfig* server_config) { @@ -143,6 +173,9 @@ Aggregated::Aggregated(const ::google::api::Service& service, sa_token_->SetAudience( auth::ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL, url_.service_control() + servicecontrol_service); + sa_token_->SetAudience( + auth::ServiceAccountToken::JWT_TOKEN_FOR_QUOTA_CONTROL, + url_.service_control() + quotacontrol_service); } } @@ -171,6 +204,7 @@ Status Aggregated::Init() { // env->StartPeriodicTimer doens't work at constructor. ServiceControlClientOptions options( GetCheckAggregationOptions(server_config_), + GetQuotaAggregationOptions(server_config_), GetReportAggregationOptions(server_config_)); std::stringstream ss; @@ -186,6 +220,11 @@ Status Aggregated::Init() { options.check_transport = [this]( const CheckRequest& request, CheckResponse* response, TransportDoneFunc on_done) { Call(request, response, on_done, nullptr); }; + + options.quota_transport = [this]( + const AllocateQuotaRequest& request, AllocateQuotaResponse* response, + TransportDoneFunc on_done) { Call(request, response, on_done, nullptr); }; + options.report_transport = [this]( const ReportRequest& request, ReportResponse* response, TransportDoneFunc on_done) { Call(request, response, on_done, nullptr); }; @@ -323,6 +362,57 @@ void Aggregated::Check( check_pool_.Free(std::move(request)); } +void Aggregated::Quota(const QuotaRequestInfo& info, + cloud_trace::CloudTraceSpan* parent_span, + std::function on_done) { + std::shared_ptr trace_span( + CreateChildSpan(parent_span, "QuotaServiceControlCache")); + + if (!client_) { + on_done(Status(Code::INTERNAL, "Missing service control client")); + return; + } + + auto request = quota_pool_.Alloc(); + + Status status = + service_control_proto_.FillAllocateQuotaRequest(info, request.get()); + if (!status.ok()) { + on_done(status); + quota_pool_.Free(std::move(request)); + return; + } + + AllocateQuotaResponse* response = new AllocateQuotaResponse(); + + auto quota_on_done = [this, response, on_done, trace_span]( + const ::google::protobuf::util::Status& status) { + TRACE(trace_span) << "AllocateQuotaRequst returned with status: " + << status.ToString(); + + if (status.ok()) { + on_done(Proto::ConvertAllocateQuotaResponse( + *response, service_control_proto_.service_name())); + } else { + on_done(Status(status.error_code(), status.error_message(), + Status::SERVICE_CONTROL)); + } + + delete response; + }; + + client_->Quota(*request, response, quota_on_done, + [trace_span, this](const AllocateQuotaRequest& request, + AllocateQuotaResponse* response, + TransportDoneFunc on_done) { + Call(request, response, on_done, trace_span.get()); + }); + + // There is no reference to request anymore at this point and it is safe to + // free request now. + quota_pool_.Free(std::move(request)); +} + Status Aggregated::GetStatistics(Statistics* esp_stat) const { if (!client_) { return Status(Code::INTERNAL, "Missing service control client"); @@ -347,13 +437,79 @@ Status Aggregated::GetStatistics(Statistics* esp_stat) const { return Status::OK; } +template +const std::string& Aggregated::GetApiReqeustUrl() { + if (typeid(RequestType) == typeid(CheckRequest)) { + return url_.check_url(); + } else if (typeid(RequestType) == typeid(AllocateQuotaRequest)) { + return url_.quota_url(); + } else { + return url_.report_url(); + } +} + +template +int Aggregated::GetHttpRequestTimeout() { + int timeout_ms = 0; + + // Set timeout on the request if it was so configured. + if (typeid(RequestType) == typeid(CheckRequest)) { + timeout_ms = kCheckDefaultTimeoutInMs; + } else if (typeid(RequestType) == typeid(AllocateQuotaRequest)) { + timeout_ms = kAllocateQuotaDefaultTimeoutInMs; + } else { + timeout_ms = kReportDefaultTimeoutInMs; + } + + if (server_config_ != nullptr && + server_config_->has_service_control_config()) { + const auto& config = server_config_->service_control_config(); + if (typeid(RequestType) == typeid(CheckRequest)) { + if (config.check_timeout_ms() > 0) { + timeout_ms = config.check_timeout_ms(); + } + } else if (typeid(RequestType) == typeid(AllocateQuotaRequest)) { + if (config.quota_timeout_ms() > 0) { + timeout_ms = config.quota_timeout_ms(); + } + } else { + if (config.report_timeout_ms() > 0) { + timeout_ms = config.report_timeout_ms(); + } + } + } + + return timeout_ms; +} + +template +const std::string& Aggregated::GetAuthToken() { + if (sa_token_) { + if (typeid(RequestType) == typeid(AllocateQuotaRequest)) { + return sa_token_->GetAuthToken( + auth::ServiceAccountToken::JWT_TOKEN_FOR_QUOTA_CONTROL); + } else { + return sa_token_->GetAuthToken( + auth::ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL); + } + } else { + static std::string empty; + return empty; + } +} + template void Aggregated::Call(const RequestType& request, ResponseType* response, TransportDoneFunc on_done, cloud_trace::CloudTraceSpan* parent_span) { std::shared_ptr trace_span( CreateChildSpan(parent_span, "Call ServiceControl server")); - std::unique_ptr http_request(new HTTPRequest([response, on_done, + + const std::string& url = GetApiReqeustUrl(); + TRACE(trace_span) << "Http request URL: " << url; + + std::unique_ptr http_request(new HTTPRequest([url, response, + on_done, trace_span, this]( Status status, std::map&&, std::string&& body) { TRACE(trace_span) << "HTTP response status: " << status.ToString(); @@ -364,9 +520,6 @@ void Aggregated::Call(const RequestType& request, ResponseType* response, Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); } } else { - const std::string& url = typeid(RequestType) == typeid(CheckRequest) - ? url_.check_url() - : url_.report_url(); env_->LogError(std::string("Failed to call ") + url + ", Error: " + status.ToString() + ", Response body: " + body); @@ -384,56 +537,25 @@ void Aggregated::Call(const RequestType& request, ResponseType* response, on_done(status.ToProto()); })); - bool is_check = (typeid(RequestType) == typeid(CheckRequest)); - const std::string& url = is_check ? url_.check_url() : url_.report_url(); - TRACE(trace_span) << "Http request URL: " << url; - std::string request_body; request.SerializeToString(&request_body); - if (!is_check && (request_body.size() > max_report_size_)) { + if ((typeid(RequestType) == typeid(ReportRequest)) && + (request_body.size() > max_report_size_)) { max_report_size_ = request_body.size(); } http_request->set_url(url) .set_method("POST") - .set_auth_token(GetAuthToken()) + .set_auth_token(GetAuthToken()) .set_header("Content-Type", application_proto) .set_body(request_body); - // Set timeout on the request if it was so configured. - if (is_check) { - http_request->set_timeout_ms(kCheckDefaultTimeoutInMs); - } else { - http_request->set_timeout_ms(kReportDefaultTimeoutInMs); - } - if (server_config_ != nullptr && - server_config_->has_service_control_config()) { - const auto& config = server_config_->service_control_config(); - if (is_check) { - if (config.check_timeout_ms() > 0) { - http_request->set_timeout_ms(config.check_timeout_ms()); - } - } else { - if (config.report_timeout_ms() > 0) { - http_request->set_timeout_ms(config.report_timeout_ms()); - } - } - } + http_request->set_timeout_ms(GetHttpRequestTimeout()); env_->RunHTTPRequest(std::move(http_request)); } -const std::string& Aggregated::GetAuthToken() { - if (sa_token_) { - return sa_token_->GetAuthToken( - auth::ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL); - } else { - static std::string empty; - return empty; - } -} - Interface* Aggregated::Create(const ::google::api::Service& service, const ServerConfig* server_config, ApiManagerEnvInterface* env, diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.h b/contrib/endpoints/src/api_manager/service_control/aggregated.h index 27e42833dbe..759cdbb41fd 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated.h +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.h @@ -23,6 +23,7 @@ #include "contrib/endpoints/src/api_manager/service_control/proto.h" #include "contrib/endpoints/src/api_manager/service_control/url.h" #include "google/api/service.pb.h" +#include "google/api/servicecontrol/v1/quota_controller.pb.h" #include "google/api/servicecontrol/v1/service_controller.pb.h" #include "include/service_control_client.h" @@ -49,6 +50,10 @@ class Aggregated : public Interface { const CheckRequestInfo& info, cloud_trace::CloudTraceSpan* parent_span, std::function on_done); + virtual void Quota(const QuotaRequestInfo& info, + cloud_trace::CloudTraceSpan* parent_span, + std::function on_done); + virtual utils::Status Init(); virtual utils::Status Close(); @@ -111,7 +116,16 @@ class Aggregated : public Interface { ::google::service_control_client::TransportDoneFunc on_done, cloud_trace::CloudTraceSpan* parent_span); - // Gets the auth token to access service control server. + // Returns API request url based on RequestType + template + const std::string& GetApiReqeustUrl(); + + // Returns API request timeout in ms based on RequestType + template + int GetHttpRequestTimeout(); + + // Returns API request auth token based on RequestType + template const std::string& GetAuthToken(); // the sevice config. @@ -134,6 +148,11 @@ class Aggregated : public Interface { // The service control client instance. std::unique_ptr<::google::service_control_client::ServiceControlClient> client_; + + // The protobuf pool to reuse AllocateQuotaRequest protobuf. + ProtoPool<::google::api::servicecontrol::v1::AllocateQuotaRequest> + quota_pool_; + // The protobuf pool to reuse CheckRequest protobuf. ProtoPool<::google::api::servicecontrol::v1::CheckRequest> check_pool_; // The protobuf pool to reuse ReportRequest protobuf. diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc b/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc index 5e9ca38a55d..c51b2b5f6aa 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc +++ b/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc @@ -19,16 +19,20 @@ #include "contrib/endpoints/src/api_manager/mock_api_manager_environment.h" #include "contrib/endpoints/src/api_manager/service_control/proto.h" #include "gmock/gmock.h" +#include "google/protobuf/text_format.h" #include "gtest/gtest.h" using ::google::api::servicecontrol::v1::CheckRequest; using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::AllocateQuotaRequest; +using ::google::api::servicecontrol::v1::AllocateQuotaResponse; using ::google::api::servicecontrol::v1::ReportRequest; using ::google::api::servicecontrol::v1::ReportResponse; using ::google::api_manager::utils::Status; using ::google::protobuf::util::error::Code; using ::google::service_control_client::ServiceControlClient; using ::google::service_control_client::TransportCheckFunc; +using ::google::service_control_client::TransportQuotaFunc; using ::google::service_control_client::TransportReportFunc; using ::testing::Return; using ::testing::Invoke; @@ -39,6 +43,39 @@ namespace api_manager { namespace service_control { namespace { + +const char kAllocateQuotaResponse[] = R"( +operation_id: "test_service" +quota_metrics { + metric_name: "serviceruntime.googleapis.com/api/consumer/quota_used_count" + metric_values { + labels { + key: "/quota_name" + value: "metric_first" + } + int64_value: 2 + } + metric_values { + labels { + key: "/quota_name" + value: "metric" + } + int64_value: 1 + } +}service_config_id: "2017-02-08r9" + +)"; + +const char kAllocateQuotaResponseErrorExhausted[] = R"( +operation_id: "test_service" +allocate_errors { + code: RESOURCE_EXHAUSTED + description: "Insufficient tokens for quota group and limit \'apiWriteQpsPerProject_LOW\' of service \'jaebonginternal.sandbox.google.com\', using the limit by ID \'container:1002409420961\'." +} +service_config_id: "2017-02-08r9" + +)"; + void FillOperationInfo(OperationInfo* op) { op->operation_id = "operation_id"; op->operation_name = "operation_name"; @@ -53,6 +90,15 @@ class MockServiceControClient : public ServiceControlClient { CheckResponse*)); MOCK_METHOD4(Check, void(const CheckRequest&, CheckResponse*, DoneCallback, TransportCheckFunc)); + + MOCK_METHOD2(Quota, + ::google::protobuf::util::Status(const AllocateQuotaRequest&, + AllocateQuotaResponse*)); + MOCK_METHOD3(Quota, void(const AllocateQuotaRequest&, AllocateQuotaResponse*, + DoneCallback)); + MOCK_METHOD4(Quota, void(const AllocateQuotaRequest&, AllocateQuotaResponse*, + DoneCallback, TransportQuotaFunc)); + MOCK_METHOD3(Report, void(const ReportRequest&, ReportResponse*, DoneCallback)); MOCK_METHOD2(Report, ::google::protobuf::util::Status(const ReportRequest&, @@ -195,6 +241,91 @@ TEST_F(AggregatedTestWithRealClient, CheckOKTest) { EXPECT_EQ(stat.send_report_operations, 0); } +class QuotaAllocationTestWithRealClient : public ::testing::Test { + public: + void SetUp() { + service_.set_name("test_service"); + service_.mutable_control()->set_environment( + "servicecontrol.googleapis.com"); + env_.reset(new ::testing::NiceMock); + sc_lib_.reset(Aggregated::Create(service_, nullptr, env_.get(), nullptr)); + ASSERT_TRUE((bool)(sc_lib_)); + // This is the call actually creating the client. + sc_lib_->Init(); + + metric_cost_vector_ = {{"metric_first", 1}, {"metric_second", 2}}; + } + + std::string getResponseBody(const char* response) { + AllocateQuotaResponse quota_response; + ::google::protobuf::TextFormat::ParseFromString(response, "a_response); + return quota_response.SerializeAsString(); + } + + void DoRunHTTPRequest(HTTPRequest* request) { + std::map headers; + + AllocateQuotaRequest quota_request; + + ASSERT_TRUE(quota_request.ParseFromString(request->body())); + ASSERT_EQ(quota_request.allocate_operation().quota_metrics_size(), 2); + + std::set> expected_costs = { + {"metric_first", 1}, {"metric_second", 2}}; + std::set> actual_costs; + + for (auto rule : quota_request.allocate_operation().quota_metrics()) { + actual_costs.insert(std::make_pair(rule.metric_name(), + rule.metric_values(0).int64_value())); + } + + ASSERT_EQ(actual_costs, expected_costs); + + request->OnComplete(Status::OK, std::move(headers), + std::move(getResponseBody(kAllocateQuotaResponse))); + } + + void DoRunHTTPRequestAllocationFailed(HTTPRequest* request) { + std::map headers; + + request->OnComplete( + Status::OK, std::move(headers), + std::move(getResponseBody(kAllocateQuotaResponseErrorExhausted))); + } + + ::google::api::Service service_; + std::unique_ptr env_; + std::unique_ptr sc_lib_; + std::vector> metric_cost_vector_; +}; + +TEST_F(QuotaAllocationTestWithRealClient, AllocateQuotaTest) { + EXPECT_CALL(*env_, DoRunHTTPRequest(_)) + .WillOnce( + Invoke(this, &QuotaAllocationTestWithRealClient::DoRunHTTPRequest)); + + QuotaRequestInfo info; + info.metric_cost_vector = &metric_cost_vector_; + + FillOperationInfo(&info); + sc_lib_->Quota(info, nullptr, + [](Status status) { ASSERT_TRUE(status.ok()); }); +} + +TEST_F(QuotaAllocationTestWithRealClient, AllocateQuotaFailedTest) { + EXPECT_CALL(*env_, DoRunHTTPRequest(_)) + .WillOnce(Invoke(this, &QuotaAllocationTestWithRealClient:: + DoRunHTTPRequestAllocationFailed)); + + QuotaRequestInfo info; + info.metric_cost_vector = &metric_cost_vector_; + + FillOperationInfo(&info); + sc_lib_->Quota(info, nullptr, [](Status status) { + ASSERT_TRUE(status.code() == Code::RESOURCE_EXHAUSTED); + }); +} + TEST(AggregatedServiceControlTest, Create) { // Verify that invalid service config yields nullptr. ::google::api::Service diff --git a/contrib/endpoints/src/api_manager/service_control/allocate_quota_response_test.cc b/contrib/endpoints/src/api_manager/service_control/allocate_quota_response_test.cc new file mode 100644 index 00000000000..729a7302527 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/allocate_quota_response_test.cc @@ -0,0 +1,184 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "contrib/endpoints/include/api_manager/utils/status.h" +#include "contrib/endpoints/src/api_manager/service_control/proto.h" +#include "gtest/gtest.h" + +namespace gasv1 = ::google::api::servicecontrol::v1; + +using ::google::api::servicecontrol::v1::QuotaError; +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { + +Status ConvertAllocateQuotaErrorToStatus(gasv1::QuotaError::Code code, + const char* error_detail, + const char* service_name) { + gasv1::AllocateQuotaResponse response; + gasv1::QuotaError* quota_error = response.add_allocate_errors(); + QuotaRequestInfo info; + quota_error->set_code(code); + quota_error->set_description(error_detail); + return Proto::ConvertAllocateQuotaResponse(response, service_name); +} + +Status ConvertAllocateQuotaErrorToStatus(gasv1::QuotaError::Code code) { + gasv1::AllocateQuotaResponse response; + std::string service_name; + response.add_allocate_errors()->set_code(code); + return Proto::ConvertAllocateQuotaResponse(response, service_name); +} + +} // namespace + +TEST(AllocateQuotaResponseTest, + AbortedWithInvalidArgumentWhenRespIsKeyInvalid) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::API_KEY_INVALID); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithInvalidArgumentWhenRespIsKeyExpired) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::API_KEY_EXPIRED); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithInvalidArgumentWhenRespIsBlockedWithResourceExausted) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::RESOURCE_EXHAUSTED); + EXPECT_EQ(Code::RESOURCE_EXHAUSTED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithInvalidArgumentWhenRespIsBlockedWithProjectSuspended) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::PROJECT_SUSPENDED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithServiceNotEnabled) { + Status result = ConvertAllocateQuotaErrorToStatus( + QuotaError::SERVICE_NOT_ENABLED, + "API api_xxxx is not enabled for the project.", "api_xxxx"); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); + EXPECT_EQ(result.message(), "API api_xxxx is not enabled for the project."); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithBillingNotActivated) { + Status result = ConvertAllocateQuotaErrorToStatus( + QuotaError::BILLING_NOT_ACTIVE, + "API api_xxxx has billing disabled. Please enable it..", "api_xxxx"); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); + EXPECT_EQ(result.message(), + "API api_xxxx has billing disabled. Please enable it."); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithIpAddressBlocked) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::IP_ADDRESS_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithRefererBlocked) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::REFERER_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithClientAppBlocked) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::CLIENT_APP_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenResponseIsBlockedWithProjectInvalid) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::PROJECT_INVALID); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithProjectDeleted) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::PROJECT_DELETED); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithApiKeyInvalid) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::API_KEY_INVALID); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithApiKeyExpiread) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::API_KEY_EXPIRED); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AcceptOKWhenRespIsBlockedWithProjectStatusUnavailable) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::PROJECT_STATUS_UNVAILABLE); + EXPECT_EQ(Code::OK, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AcceptOKWhenRespIsBlockedWithServiceStatusUnavailable) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::SERVICE_STATUS_UNAVAILABLE); + EXPECT_EQ(Code::OK, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AcceptOKWhenRespIsBlockedWithBillingStatusUnavailable) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::BILLING_STATUS_UNAVAILABLE); + EXPECT_EQ(Code::OK, result.code()); +} + +TEST(AllocateQuotaResponseTest, FailOpenWhenResponseIsUnknownBillingStatus) { + EXPECT_TRUE( + ConvertAllocateQuotaErrorToStatus(QuotaError::BILLING_STATUS_UNAVAILABLE) + .ok()); +} + +TEST(AllocateQuotaResponseTest, FailOpenWhenResponseIsUnknownServiceStatus) { + EXPECT_TRUE( + ConvertAllocateQuotaErrorToStatus(QuotaError::SERVICE_STATUS_UNAVAILABLE) + .ok()); +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/info.h b/contrib/endpoints/src/api_manager/service_control/info.h index f203057cc9e..184d598dae7 100644 --- a/contrib/endpoints/src/api_manager/service_control/info.h +++ b/contrib/endpoints/src/api_manager/service_control/info.h @@ -17,7 +17,8 @@ #include "google/protobuf/stubs/stringpiece.h" -#include +#include "google/api/quota.pb.h" + #include #include #include @@ -74,6 +75,11 @@ struct CheckRequestInfo : public OperationInfo { // Whether the method allow unregistered calls. bool allow_unregistered_calls; + // used for api key restriction check + std::string android_package_name; + std::string android_cert_fingerprint; + std::string ios_bundle_id; + CheckRequestInfo() : allow_unregistered_calls(false) {} }; @@ -89,6 +95,12 @@ struct CheckResponseInfo { CheckResponseInfo() : is_api_key_valid(true), service_is_activated(true) {} }; +struct QuotaRequestInfo : public OperationInfo { + std::string method_name; + + const std::vector>* metric_cost_vector; +}; + // Information to fill Report request protobuf. struct ReportRequestInfo : public OperationInfo { // The HTTP response code. diff --git a/contrib/endpoints/src/api_manager/service_control/interface.h b/contrib/endpoints/src/api_manager/service_control/interface.h index 708acc56a88..a6188e0f73b 100644 --- a/contrib/endpoints/src/api_manager/service_control/interface.h +++ b/contrib/endpoints/src/api_manager/service_control/interface.h @@ -70,6 +70,17 @@ class Interface { const CheckRequestInfo& info, cloud_trace::CloudTraceSpan* parent_span, std::function on_done) = 0; + // on_done() function will be called once it is completed. + // utils::Status in the on_done callback: + // If status.code is more than 100, it is the HTTP response status + // from the service control server. + // If status code is less than 20, within the ranges defined by + // google/protobuf/stubs/status.h, is from parsing error response + // body. + virtual void Quota(const QuotaRequestInfo& info, + cloud_trace::CloudTraceSpan* parent_span, + std::function on_done) = 0; + // Get statistics of ServiceControl library. virtual utils::Status GetStatistics(Statistics* stat) const = 0; }; diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 694f299d22c..aa88fb39eb2 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -30,6 +30,7 @@ #include "utils/distribution_helper.h" using ::google::api::servicecontrol::v1::CheckError; +using ::google::api::servicecontrol::v1::QuotaError; using ::google::api::servicecontrol::v1::CheckRequest; using ::google::api::servicecontrol::v1::CheckResponse; using ::google::api::servicecontrol::v1::Distribution; @@ -49,6 +50,11 @@ namespace google { namespace api_manager { namespace service_control { +const char kConsumerQuotaUsedCount[] = + "serviceruntime.googleapis.com/api/consumer/quota_used_count"; + +const char kQuotaName[] = "/quota_name"; + struct SupportedMetric { const char* name; ::google::api::MetricDescriptor_MetricKind metric_kind; @@ -420,6 +426,12 @@ const char kServiceControlServiceAgent[] = const char kServiceControlUserAgent[] = "servicecontrol.googleapis.com/user_agent"; const char kServiceControlPlatform[] = "servicecontrol.googleapis.com/platform"; +const char kServiceControlAndroidPackageName[] = + "servicecontrol.googleapis.com/android_package_name"; +const char kServiceControlAndroidCertFingerprint[] = + "servicecontrol.googleapis.com/android_cert_fingerprint"; +const char kServiceControlIosBundleId[] = + "servicecontrol.googleapis.com/ios_bundle_id"; // User agent label value // The value for kUserAgent should be configured at service control server. @@ -905,6 +917,62 @@ Proto::Proto(const std::set& logs, service_name_(service_name), service_config_id_(service_config_id) {} +utils::Status Proto::FillAllocateQuotaRequest( + const QuotaRequestInfo& info, + ::google::api::servicecontrol::v1::AllocateQuotaRequest* request) { + ::google::api::servicecontrol::v1::QuotaOperation* operation = + request->mutable_allocate_operation(); + + // service_name + request->set_service_name(service_name_); + // service_config_id + request->set_service_config_id(service_config_id_); + + // allocate_operation.operation_id + if (!info.operation_id.empty()) { + operation->set_operation_id(info.operation_id); + } + // allocate_operation.method_name + if (!info.method_name.empty()) { + operation->set_method_name(info.method_name); + } + // allocate_operation.consumer_id + if (!info.api_key.empty()) { + operation->set_consumer_id(std::string(kConsumerIdApiKey) + + std::string(info.api_key)); + } + + // allocate_operation.quota_mode + operation->set_quota_mode( + ::google::api::servicecontrol::v1::QuotaOperation_QuotaMode:: + QuotaOperation_QuotaMode_NORMAL); + + // allocate_operation.labels + auto* labels = operation->mutable_labels(); + if (!info.client_ip.empty()) { + (*labels)[kServiceControlCallerIp] = info.client_ip; + } + + if (!info.referer.empty()) { + (*labels)[kServiceControlReferer] = info.referer; + } + (*labels)[kServiceControlUserAgent] = kUserAgent; + (*labels)[kServiceControlServiceAgent] = + kServiceAgentPrefix + utils::Version::instance().get(); + + if (info.metric_cost_vector) { + for (auto metric : *info.metric_cost_vector) { + MetricValueSet* value_set = operation->add_quota_metrics(); + value_set->set_metric_name(metric.first); + MetricValue* value = value_set->add_metric_values(); + const auto& cost = metric.second; + value->set_int64_value(cost <= 0 ? 1 : cost); + } + } + + return Status::OK; +} + Status Proto::FillCheckRequest(const CheckRequestInfo& info, CheckRequest* request) { Status status = VerifyRequiredCheckFields(info); @@ -928,6 +996,18 @@ Status Proto::FillCheckRequest(const CheckRequestInfo& info, (*labels)[kServiceControlUserAgent] = kUserAgent; (*labels)[kServiceControlServiceAgent] = kServiceAgentPrefix + utils::Version::instance().get(); + + if (!info.android_package_name.empty()) { + (*labels)[kServiceControlAndroidPackageName] = info.android_package_name; + } + if (!info.android_cert_fingerprint.empty()) { + (*labels)[kServiceControlAndroidCertFingerprint] = + info.android_cert_fingerprint; + } + if (!info.ios_bundle_id.empty()) { + (*labels)[kServiceControlIosBundleId] = info.ios_bundle_id; + } + return Status::OK; } @@ -992,6 +1072,100 @@ Status Proto::FillReportRequest(const ReportRequestInfo& info, return Status::OK; } +Status Proto::ConvertAllocateQuotaResponse( + const ::google::api::servicecontrol::v1::AllocateQuotaResponse& response, + const std::string& service_name) { + // response.operation_id() + if (response.allocate_errors().size() == 0) { + return Status::OK; + } + + const ::google::api::servicecontrol::v1::QuotaError& error = + response.allocate_errors().Get(0); + + switch (error.code()) { + case ::google::api::servicecontrol::v1::QuotaError::UNSPECIFIED: + // This is never used. + break; + + case ::google::api::servicecontrol::v1::QuotaError::RESOURCE_EXHAUSTED: + // Quota allocation failed. + // Same as [google.rpc.Code.RESOURCE_EXHAUSTED][]. + return Status(Code::RESOURCE_EXHAUSTED, "Quota allocation failed."); + + case ::google::api::servicecontrol::v1::QuotaError::PROJECT_SUSPENDED: + // Consumer project has been suspended. + return Status(Code::PERMISSION_DENIED, "Project suspended."); + + case ::google::api::servicecontrol::v1::QuotaError::SERVICE_NOT_ENABLED: + // Consumer has not enabled the service. + return Status(Code::PERMISSION_DENIED, + std::string("API ") + service_name + + " is not enabled for the project."); + + case ::google::api::servicecontrol::v1::QuotaError::BILLING_NOT_ACTIVE: + // Consumer cannot access the service because billing is disabled. + return Status(Code::PERMISSION_DENIED, + std::string("API ") + service_name + + " has billing disabled. Please enable it."); + + case ::google::api::servicecontrol::v1::QuotaError::PROJECT_DELETED: + // Consumer's project has been marked as deleted (soft deletion). + case ::google::api::servicecontrol::v1::QuotaError::PROJECT_INVALID: + // Consumer's project number or ID does not represent a valid project. + return Status(Code::INVALID_ARGUMENT, + "Client project not valid. Please pass a valid project."); + + case ::google::api::servicecontrol::v1::QuotaError::IP_ADDRESS_BLOCKED: + // IP address of the consumer is invalid for the specific consumer + // project. + return Status(Code::PERMISSION_DENIED, "IP address blocked."); + + case ::google::api::servicecontrol::v1::QuotaError::REFERER_BLOCKED: + // Referer address of the consumer request is invalid for the specific + // consumer project. + return Status(Code::PERMISSION_DENIED, "Referer blocked."); + + case ::google::api::servicecontrol::v1::QuotaError::CLIENT_APP_BLOCKED: + // Client application of the consumer request is invalid for the + // specific consumer project. + return Status(Code::PERMISSION_DENIED, "Client app blocked."); + + case ::google::api::servicecontrol::v1::QuotaError::API_KEY_INVALID: + // Specified API key is invalid. + return Status(Code::INVALID_ARGUMENT, + "API key not valid. Please pass a valid API key."); + + case ::google::api::servicecontrol::v1::QuotaError::API_KEY_EXPIRED: + // Specified API Key has expired. + return Status(Code::INVALID_ARGUMENT, + "API key expired. Please renew the API key."); + + case ::google::api::servicecontrol::v1::QuotaError:: + PROJECT_STATUS_UNVAILABLE: + // The backend server for looking up project id/number is unavailable. + case ::google::api::servicecontrol::v1::QuotaError:: + SERVICE_STATUS_UNAVAILABLE: + // The backend server for checking service status is unavailable. + case ::google::api::servicecontrol::v1::QuotaError:: + BILLING_STATUS_UNAVAILABLE: + // The backend server for checking billing status is unavailable. + // Fail open for internal server errors per recommendation + case ::google::api::servicecontrol::v1::QuotaError:: + QUOTA_SYSTEM_UNAVAILABLE: + // The backend server for checking quota limits is unavailable. + return Status::OK; + + default: + return Status( + Code::INTERNAL, + std::string("Request blocked due to unsupported error code: ") + + std::to_string(error.code())); + } + + return Status::OK; +} + Status Proto::ConvertCheckResponse(const CheckResponse& check_response, const std::string& service_name, CheckResponseInfo* check_response_info) { diff --git a/contrib/endpoints/src/api_manager/service_control/proto.h b/contrib/endpoints/src/api_manager/service_control/proto.h index bea2a948b63..d4fa09592c9 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.h +++ b/contrib/endpoints/src/api_manager/service_control/proto.h @@ -19,6 +19,7 @@ #include "contrib/endpoints/src/api_manager/service_control/info.h" #include "google/api/label.pb.h" #include "google/api/metric.pb.h" +#include "google/api/servicecontrol/v1/quota_controller.pb.h" #include "google/api/servicecontrol/v1/service_controller.pb.h" namespace google { @@ -48,6 +49,10 @@ class Proto final { const CheckRequestInfo& info, ::google::api::servicecontrol::v1::CheckRequest* request); + utils::Status FillAllocateQuotaRequest( + const QuotaRequestInfo& info, + ::google::api::servicecontrol::v1::AllocateQuotaRequest* request); + // Fills the CheckRequest protobuf from info. // FillReportRequest function should copy the strings pointed by info. // These buffers may be freed after the FillReportRequest call. @@ -64,6 +69,10 @@ class Proto final { const ::google::api::servicecontrol::v1::CheckResponse& response, const std::string& service_name, CheckResponseInfo* check_response_info); + static utils::Status ConvertAllocateQuotaResponse( + const ::google::api::servicecontrol::v1::AllocateQuotaResponse& response, + const std::string& service_name); + static bool IsMetricSupported(const ::google::api::MetricDescriptor& metric); static bool IsLabelSupported(const ::google::api::LabelDescriptor& label); const std::string& service_name() const { return service_name_; } diff --git a/contrib/endpoints/src/api_manager/service_control/proto_test.cc b/contrib/endpoints/src/api_manager/service_control/proto_test.cc index 609cb54abe7..4e2a0aeb61b 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto_test.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto_test.cc @@ -76,6 +76,12 @@ void FillCheckRequestInfo(CheckRequestInfo* request) { request->referer = "referer"; } +void FillAllocateQuotaRequestInfo(QuotaRequestInfo* request) { + request->client_ip = "1.2.3.4"; + request->referer = "referer"; + request->method_name = "operation_name"; +} + void FillReportRequestInfo(ReportRequestInfo* request) { request->referer = "referer"; request->response_code = 200; @@ -122,6 +128,12 @@ std::string CheckRequestToString(gasv1::CheckRequest* request) { return text; } +std::string AllocateQuotaRequestToString(gasv1::AllocateQuotaRequest* request) { + std::string text; + google::protobuf::TextFormat::PrintToString(*request, &text); + return text; +} + std::string ReportRequestToString(gasv1::ReportRequest* request) { gasv1::Operation* op = request->mutable_operations(0); SetFixTimeStamps(op); @@ -161,6 +173,62 @@ TEST_F(ProtoTest, FillGoodCheckRequestTest) { ASSERT_EQ(expected_text, text); } +TEST_F(ProtoTest, FillGoodCheckRequestAndroidIosTest) { + CheckRequestInfo info; + FillOperationInfo(&info); + FillCheckRequestInfo(&info); + + info.android_package_name = "com.google.cloud"; + info.android_cert_fingerprint = "AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0"; + info.ios_bundle_id = "5b40ad6af9a806305a0a56d7cb91b82a27c26909"; + + gasv1::CheckRequest request; + ASSERT_TRUE(scp_.FillCheckRequest(info, &request).ok()); + + std::string text = CheckRequestToString(&request); + std::string expected_text = + ReadTestBaseline("check_request_android_ios.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillGoodAllocateQuotaRequestTest) { + std::vector> metric_cost_vector = { + {"metric_first", 1}, {"metric_second", 2}}; + + google::api_manager::service_control::QuotaRequestInfo info; + info.metric_cost_vector = &metric_cost_vector; + + FillOperationInfo(&info); + FillAllocateQuotaRequestInfo(&info); + + gasv1::AllocateQuotaRequest request; + ASSERT_TRUE(scp_.FillAllocateQuotaRequest(info, &request).ok()); + + std::string text = AllocateQuotaRequestToString(&request); + std::string expected_text = ReadTestBaseline("allocate_quota_request.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillAllocateQuotaRequestNoMethodNameTest) { + std::vector> metric_cost_vector = { + {"metric_first", 1}, {"metric_second", 2}}; + + google::api_manager::service_control::QuotaRequestInfo info; + FillOperationInfo(&info); + info.metric_cost_vector = &metric_cost_vector; + info.client_ip = "1.2.3.4"; + info.referer = "referer"; + info.method_name = ""; + + gasv1::AllocateQuotaRequest request; + ASSERT_TRUE(scp_.FillAllocateQuotaRequest(info, &request).ok()); + + std::string text = AllocateQuotaRequestToString(&request); + std::string expected_text = + ReadTestBaseline("allocate_quota_request_no_method_name.golden"); + ASSERT_EQ(expected_text, text); +} + TEST_F(ProtoTest, FillNoApiKeyCheckRequestTest) { CheckRequestInfo info; info.operation_id = "operation_id"; diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request.golden new file mode 100644 index 00000000000..73a26239238 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request.golden @@ -0,0 +1,36 @@ +service_name: "test_service" +allocate_operation { + operation_id: "operation_id" + method_name: "operation_name" + consumer_id: "api_key:api_key_x" + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + quota_metrics { + metric_name: "metric_first" + metric_values { + int64_value: 1 + } + } + quota_metrics { + metric_name: "metric_second" + metric_values { + int64_value: 2 + } + } + quota_mode: NORMAL +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_android_ios.golden b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_android_ios.golden new file mode 100644 index 00000000000..476bae4e5f5 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_android_ios.golden @@ -0,0 +1,48 @@ +service_name: "test_service" +allocate_operation { + operation_id: "operation_id" + method_name: "operation_name" + consumer_id: "api_key:api_key_x" + labels { + key: "servicecontrol.googleapis.com/android_cert_fingerprint" + value: "AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0" + } + labels { + key: "servicecontrol.googleapis.com/android_package_name" + value: "com.google.cloud" + } + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/ios_bundle_id" + value: "5b40ad6af9a806305a0a56d7cb91b82a27c26909" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + quota_metrics { + metric_name: "metric_first" + metric_values { + int64_value: 1 + } + } + quota_metrics { + metric_name: "metric_second" + metric_values { + int64_value: 2 + } + } + quota_mode: NORMAL +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_no_method_name.golden b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_no_method_name.golden new file mode 100644 index 00000000000..34a59f47652 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_no_method_name.golden @@ -0,0 +1,35 @@ +service_name: "test_service" +allocate_operation { + operation_id: "operation_id" + consumer_id: "api_key:api_key_x" + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + quota_metrics { + metric_name: "metric_first" + metric_values { + int64_value: 1 + } + } + quota_metrics { + metric_name: "metric_second" + metric_values { + int64_value: 2 + } + } + quota_mode: NORMAL +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/check_request_android_ios.golden b/contrib/endpoints/src/api_manager/service_control/testdata/check_request_android_ios.golden new file mode 100644 index 00000000000..c847061da4f --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/check_request_android_ios.golden @@ -0,0 +1,43 @@ +service_name: "test_service" +operation { + operation_id: "operation_id" + operation_name: "operation_name" + consumer_id: "api_key:api_key_x" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "servicecontrol.googleapis.com/android_cert_fingerprint" + value: "AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0" + } + labels { + key: "servicecontrol.googleapis.com/android_package_name" + value: "com.google.cloud" + } + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/ios_bundle_id" + value: "5b40ad6af9a806305a0a56d7cb91b82a27c26909" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/url.cc b/contrib/endpoints/src/api_manager/service_control/url.cc index d113d402d1f..f106ccd40fa 100644 --- a/contrib/endpoints/src/api_manager/service_control/url.cc +++ b/contrib/endpoints/src/api_manager/service_control/url.cc @@ -27,6 +27,7 @@ namespace { // /v1/services/{service}:report const char v1_services_path[] = "/v1/services/"; const char check_verb[] = ":check"; +const char quota_verb[] = ":allocateQuota"; const char report_verb[] = ":report"; const char http[] = "http://"; const char https[] = "https://"; @@ -66,6 +67,7 @@ Url::Url(const ::google::api::Service* service, std::string path = service_control_ + v1_services_path + service->name(); check_url_ = path + check_verb; report_url_ = path + report_verb; + quota_url_ = path + quota_verb; } } diff --git a/contrib/endpoints/src/api_manager/service_control/url.h b/contrib/endpoints/src/api_manager/service_control/url.h index 4615864bbdd..fb17aa5e006 100644 --- a/contrib/endpoints/src/api_manager/service_control/url.h +++ b/contrib/endpoints/src/api_manager/service_control/url.h @@ -31,12 +31,14 @@ class Url { // Pre-computed url for service control. const std::string& service_control() const { return service_control_; } const std::string& check_url() const { return check_url_; } + const std::string& quota_url() const { return quota_url_; } const std::string& report_url() const { return report_url_; } private: // Pre-computed url for service control methods. std::string service_control_; std::string check_url_; + std::string quota_url_; std::string report_url_; }; diff --git a/contrib/endpoints/src/api_manager/service_control/url_test.cc b/contrib/endpoints/src/api_manager/service_control/url_test.cc index afbf48bd154..8e0bb2d188a 100644 --- a/contrib/endpoints/src/api_manager/service_control/url_test.cc +++ b/contrib/endpoints/src/api_manager/service_control/url_test.cc @@ -57,6 +57,10 @@ TEST(UrlTest, PrependHttps) { ASSERT_EQ( "https://servicecontrol.googleapis.com/v1/services/https-config:report", url.report_url()); + ASSERT_EQ( + "https://servicecontrol.googleapis.com/v1/services/" + "https-config:allocateQuota", + url.quota_url()); } TEST(UrlTest, ServerControlOverride) { diff --git a/contrib/endpoints/src/grpc/transcoding/BUILD b/contrib/endpoints/src/grpc/transcoding/BUILD index 894fd27b960..dce09d218c4 100644 --- a/contrib/endpoints/src/grpc/transcoding/BUILD +++ b/contrib/endpoints/src/grpc/transcoding/BUILD @@ -67,6 +67,7 @@ cc_library( "message_stream.h", ], deps = [ + ":transcoder_input_stream", "//external:protobuf", ], ) @@ -125,6 +126,7 @@ cc_library( "message_reader.h", ], deps = [ + ":transcoder_input_stream", "//external:protobuf", ], ) @@ -144,6 +146,17 @@ cc_library( ], ) +cc_library( + name = "transcoder_input_stream", + srcs = [ + "transcoder_input_stream.h", + ], + visibility = ["//visibility:public"], + deps = [ + "@protobuf_git//:protobuf", + ], +) + cc_library( name = "transcoding", srcs = [ @@ -223,6 +236,7 @@ cc_library( srcs = ["test_common.cc"], hdrs = ["test_common.h"], deps = [ + ":transcoder_input_stream", "//external:googletest", "//external:protobuf", "//external:service_config", diff --git a/contrib/endpoints/src/grpc/transcoding/message_reader.cc b/contrib/endpoints/src/grpc/transcoding/message_reader.cc index e5e4aeeae14..23b6a0cbad3 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_reader.cc +++ b/contrib/endpoints/src/grpc/transcoding/message_reader.cc @@ -18,7 +18,6 @@ #include -#include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl.h" namespace google { @@ -29,7 +28,7 @@ namespace transcoding { namespace pb = ::google::protobuf; namespace pbio = ::google::protobuf::io; -MessageReader::MessageReader(pbio::ZeroCopyInputStream* in) +MessageReader::MessageReader(TranscoderInputStream* in) : in_(in), current_message_size_(0), have_current_message_size_(false), @@ -99,7 +98,7 @@ std::unique_ptr MessageReader::NextMessage() { // Check if we have the current message size. If not try to read it. if (!have_current_message_size_) { const size_t kDelimiterSize = 5; - if (in_->ByteCount() < static_cast(kDelimiterSize)) { + if (in_->BytesAvailable() < static_cast(kDelimiterSize)) { // We don't have 5 bytes available to read the length of the message. // Find out whether the stream is finished and return false. finished_ = IsStreamFinished(in_); @@ -117,10 +116,7 @@ std::unique_ptr MessageReader::NextMessage() { have_current_message_size_ = true; } - // We interpret ZeroCopyInputStream::ByteCount() as the number of bytes - // available for reading at the moment. Check if we have the full message - // available to read. - if (in_->ByteCount() < static_cast(current_message_size_)) { + if (in_->BytesAvailable() < static_cast(current_message_size_)) { // We don't have a full message return std::unique_ptr(); } diff --git a/contrib/endpoints/src/grpc/transcoding/message_reader.h b/contrib/endpoints/src/grpc/transcoding/message_reader.h index df175561f47..f07408e23af 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_reader.h +++ b/contrib/endpoints/src/grpc/transcoding/message_reader.h @@ -17,7 +17,7 @@ #include -#include "google/protobuf/io/zero_copy_stream.h" +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/protobuf/stubs/status.h" namespace google { @@ -49,11 +49,6 @@ namespace transcoding { // } // } // -// NOTE: MesssageReader assumes that ZeroCopyInputStream::ByteCount() returns -// the number of bytes available to read at the moment. That's what -// MessageReader uses to determine whether there is a complete message -// available or not. -// // NOTE: MessageReader is unable to recognize the case when there is an // incomplete message at the end of the input. The callers will need to // detect it and act appropriately. @@ -64,7 +59,7 @@ namespace transcoding { // class MessageReader { public: - MessageReader(::google::protobuf::io::ZeroCopyInputStream* in); + MessageReader(TranscoderInputStream* in); // If a full message is available, NextMessage() returns a ZeroCopyInputStream // over the message. Otherwise returns nullptr - this might be temporary, the @@ -82,7 +77,7 @@ class MessageReader { bool Finished() const { return finished_; } private: - ::google::protobuf::io::ZeroCopyInputStream* in_; + TranscoderInputStream* in_; // The size of the current message. unsigned int current_message_size_; // Whether we have read the current message size or not diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream.cc b/contrib/endpoints/src/grpc/transcoding/message_stream.cc index 11af66bb170..26a09a23361 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_stream.cc +++ b/contrib/endpoints/src/grpc/transcoding/message_stream.cc @@ -19,7 +19,6 @@ #include #include -#include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" namespace google { @@ -32,12 +31,12 @@ namespace pbio = ::google::protobuf::io; namespace { // a ZeroCopyInputStream implementation over a MessageStream implementation -class ZeroCopyStreamOverMessageStream : public pbio::ZeroCopyInputStream { +class InputStreamOverMessageStream : public TranscoderInputStream { public: - // src - the underlying MessageStream. ZeroCopyStreamOverMessageStream doesn't + // src - the underlying MessageStream. InputStreamOverMessageStream doesn't // maintain the ownership of src, the caller must make sure it exists - // throughtout the lifetime of ZeroCopyStreamOverMessageStream. - ZeroCopyStreamOverMessageStream(MessageStream* src) + // throughtout the lifetime of InputStreamOverMessageStream. + InputStreamOverMessageStream(MessageStream* src) : src_(src), message_(), position_(0) {} // ZeroCopyInputStream implementation @@ -72,19 +71,15 @@ class ZeroCopyStreamOverMessageStream : public pbio::ZeroCopyInputStream { bool Skip(int) { return false; } // Not implemented (no need) - ::google::protobuf::int64 ByteCount() const { - // NOTE: we are changing the ByteCount() interpretation. In our case - // ByteCount() returns the number of bytes available for reading at this - // moment. In the original interpretation it is supposed to be the number - // of bytes read so far. - // We need this such that the consumers are able to read the gRPC delimited - // message stream only if there is a full message available. + google::protobuf::int64 ByteCount() const { return 0; } // Not implemented + + int64_t BytesAvailable() const { if (position_ >= message_.size()) { // If the current message is all done, try to read the next message // to make sure we return the correct byte count. - const_cast(this)->ReadNextMessage(); + const_cast(this)->ReadNextMessage(); } - return static_cast<::google::protobuf::int64>(message_.size() - position_); + return static_cast(message_.size() - position_); } private: @@ -109,10 +104,9 @@ class ZeroCopyStreamOverMessageStream : public pbio::ZeroCopyInputStream { } // namespace -std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> -MessageStream::CreateZeroCopyInputStream() { - return std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream>( - new ZeroCopyStreamOverMessageStream(this)); +std::unique_ptr MessageStream::CreateInputStream() { + return std::unique_ptr( + new InputStreamOverMessageStream(this)); } } // namespace transcoding diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream.h b/contrib/endpoints/src/grpc/transcoding/message_stream.h index 435040332a7..aef90f524f6 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_stream.h +++ b/contrib/endpoints/src/grpc/transcoding/message_stream.h @@ -18,6 +18,7 @@ #include #include +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/stubs/status.h" @@ -73,8 +74,7 @@ class MessageStream { // Virtual destructor virtual ~MessageStream() {} // Creates ZeroCopyInputStream implementation based on this stream - std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> - CreateZeroCopyInputStream(); + std::unique_ptr CreateInputStream(); }; } // namespace transcoding diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc b/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc index 3699b499bfc..81276310f01 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc +++ b/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc @@ -70,14 +70,14 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { bool Test(const Messages& messages) { TestMessageStream test_message_stream; - auto zero_copy_stream = test_message_stream.CreateZeroCopyInputStream(); + auto input_stream = test_message_stream.CreateInputStream(); const void* data = nullptr; int size = 0; // Check that Next() returns true and a 0-sized buffer meaning that // nothing is available at the moment. - if (!zero_copy_stream->Next(&data, &size)) { + if (!input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; return false; } @@ -91,13 +91,13 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { test_message_stream.AddMessage(message); // message.size() bytes must be available for reading - if (static_cast(message.size()) != zero_copy_stream->ByteCount()) { - EXPECT_EQ(message.size(), zero_copy_stream->ByteCount()); + if (static_cast(message.size()) != input_stream->BytesAvailable()) { + EXPECT_EQ(message.size(), input_stream->BytesAvailable()); return false; } // Now try to read & match the message - if (!zero_copy_stream->Next(&data, &size)) { + if (!input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; return false; } @@ -120,16 +120,16 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { // Not a valid test case continue; } - zero_copy_stream->BackUp(backup_size); + input_stream->BackUp(backup_size); // backup_size bytes must be available for reading again - if (static_cast(backup_size) != zero_copy_stream->ByteCount()) { - EXPECT_EQ(message.size(), zero_copy_stream->ByteCount()); + if (static_cast(backup_size) != input_stream->BytesAvailable()) { + EXPECT_EQ(message.size(), input_stream->BytesAvailable()); return false; } // Now Next() must return the backed up data again. - if (!zero_copy_stream->Next(&data, &size)) { + if (!input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; return false; } @@ -143,7 +143,7 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { } // At this point no data should be available - if (!zero_copy_stream->Next(&data, &size)) { + if (!input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; return false; } @@ -156,7 +156,7 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { // Now finish the MessageStream & make sure the ZeroCopyInputStream has // ended. test_message_stream.Finish(); - if (zero_copy_stream->Next(&data, &size)) { + if (input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream still hasn't finished" << std::endl; return false; } @@ -201,14 +201,14 @@ TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DifferenteSizesOneStream) { TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { TestMessageStream test_message_stream; - auto zero_copy_stream = test_message_stream.CreateZeroCopyInputStream(); + auto input_stream = test_message_stream.CreateInputStream(); const void* data = nullptr; int size = 0; // Check that Next() returns true and a 0-sized buffer meaning that // nothing is available at the moment. - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(0, size); // Test messages @@ -221,16 +221,16 @@ TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { test_message_stream.AddMessage(message1); // message1 is available for reading - EXPECT_EQ(message1.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message1.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message1, std::string(reinterpret_cast(data), size)); // Back up a bit - zero_copy_stream->BackUp(5); + input_stream->BackUp(5); // Now read the backed up data again - EXPECT_EQ(5, zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(5, input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message1.substr(message1.size() - 5), std::string(reinterpret_cast(data), size)); @@ -238,20 +238,20 @@ TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { test_message_stream.AddMessage(message2); // message2 is available for reading - EXPECT_EQ(message2.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message2.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message2, std::string(reinterpret_cast(data), size)); // Back up all of message2 - zero_copy_stream->BackUp(message2.size()); + input_stream->BackUp(message2.size()); // Now read message2 again - EXPECT_EQ(message2.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message2.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message2, std::string(reinterpret_cast(data), size)); // At this point no data should be available - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(0, size); // Add both message3 & message4 & finish the MessageStream afterwards @@ -260,16 +260,16 @@ TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { test_message_stream.Finish(); // Read & match both message3 & message4 - EXPECT_EQ(message3.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message3.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message3, std::string(reinterpret_cast(data), size)); - EXPECT_EQ(message4.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message4.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message4, std::string(reinterpret_cast(data), size)); // All done! - EXPECT_FALSE(zero_copy_stream->Next(&data, &size)); + EXPECT_FALSE(input_stream->Next(&data, &size)); } } // namespace diff --git a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc index eb5e54f9bec..8354cc07f25 100644 --- a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc +++ b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc @@ -18,7 +18,6 @@ #include -#include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" #include "google/protobuf/stubs/status.h" #include "google/protobuf/util/json_util.h" @@ -31,7 +30,7 @@ namespace transcoding { ResponseToJsonTranslator::ResponseToJsonTranslator( ::google::protobuf::util::TypeResolver* type_resolver, std::string type_url, - bool streaming, ::google::protobuf::io::ZeroCopyInputStream* in) + bool streaming, TranscoderInputStream* in) : type_resolver_(type_resolver), type_url_(std::move(type_url)), streaming_(streaming), diff --git a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h index e41791467f8..d674d71d258 100644 --- a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h +++ b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h @@ -66,8 +66,7 @@ class ResponseToJsonTranslator : public MessageStream { // format (http://www.grpc.io/docs/guides/wire.html) ResponseToJsonTranslator( ::google::protobuf::util::TypeResolver* type_resolver, - std::string type_url, bool streaming, - ::google::protobuf::io::ZeroCopyInputStream* in); + std::string type_url, bool streaming, TranscoderInputStream* in); // MessageStream implementation bool NextMessage(std::string* message); diff --git a/contrib/endpoints/src/grpc/transcoding/test_common.cc b/contrib/endpoints/src/grpc/transcoding/test_common.cc index 269bc1172d7..12a278dbfa3 100644 --- a/contrib/endpoints/src/grpc/transcoding/test_common.cc +++ b/contrib/endpoints/src/grpc/transcoding/test_common.cc @@ -81,7 +81,7 @@ void TestZeroCopyInputStream::BackUp(int count) { position_ -= count; } -pb::int64 TestZeroCopyInputStream::ByteCount() const { +int64_t TestZeroCopyInputStream::BytesAvailable() const { auto total = current_.size() - position_; for (auto chunk : chunks_) { total += chunk.size(); diff --git a/contrib/endpoints/src/grpc/transcoding/test_common.h b/contrib/endpoints/src/grpc/transcoding/test_common.h index bc452e052fc..b6f2532b867 100644 --- a/contrib/endpoints/src/grpc/transcoding/test_common.h +++ b/contrib/endpoints/src/grpc/transcoding/test_common.h @@ -20,6 +20,7 @@ #include #include +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/api/service.pb.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/text_format.h" @@ -33,8 +34,7 @@ namespace testing { // An implementation of ZeroCopyInputStream for testing. // The tests define the chunks that TestZeroCopyInputStream produces. -class TestZeroCopyInputStream - : public ::google::protobuf::io::ZeroCopyInputStream { +class TestZeroCopyInputStream : public TranscoderInputStream { public: TestZeroCopyInputStream(); @@ -50,8 +50,9 @@ class TestZeroCopyInputStream // ZeroCopyInputStream methods bool Next(const void** data, int* size); void BackUp(int count); - ::google::protobuf::int64 ByteCount() const; - bool Skip(int) { return false; } // Not implemented + int64_t BytesAvailable() const; + ::google::protobuf::int64 ByteCount() const { return 0; } // Not implemented + bool Skip(int) { return false; } // Not implemented private: std::deque chunks_; diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder.h b/contrib/endpoints/src/grpc/transcoding/transcoder.h index da58386089d..806c519f921 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder.h +++ b/contrib/endpoints/src/grpc/transcoding/transcoder.h @@ -15,7 +15,7 @@ #ifndef GRPC_TRANSCODING_TRANSCODER_H_ #define GRPC_TRANSCODING_TRANSCODER_H_ -#include "google/protobuf/io/zero_copy_stream.h" +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/protobuf/stubs/status.h" namespace google { @@ -28,11 +28,11 @@ namespace transcoding { // - translated response stream, // - status of response translation. // -// NOTE: Transcoder uses ::google::protobuf::io::ZeroCopyInputStream for -// carrying the payloads both for input and output. It assumes the -// following interpretation of the ZeroCopyInputStream interface: +// NOTE: Transcoder uses TranscoderInputStream for carrying the payloads +// both for input and output. It assumes the following interpretation +// of the TranscoderInputStream interface: // -// bool ZeroCopyInputStream::Next(const void** data, int* size); +// bool TranscoderInputStream::Next(const void** data, int* size); // // Obtains a chunk of data from the stream. // @@ -52,7 +52,7 @@ namespace transcoding { // again later. // // -// void ZeroCopyInputStream::BackUp(int count); +// void TranscoderInputStream::BackUp(int count); // // Backs up a number of bytes, so that the next call to Next() returns // data again that was already returned by the last call to Next(). This @@ -72,12 +72,12 @@ namespace transcoding { // the same data again before producing new data. // // -// bool ZeroCopyInputStream::Skip(int count); +// bool TranscoderInputStream::Skip(int count); // // Not used and not implemented by the Transcoder. // // -// int64 ZeroCopyInputStream::ByteCount() const; +// int64_t TranscoderInputStream::BytesAvailable() const; // // Returns the number of bytes available for reading at this moment // @@ -133,7 +133,7 @@ namespace transcoding { class Transcoder { public: // ZeroCopyInputStream to read the transcoded request. - virtual ::google::protobuf::io::ZeroCopyInputStream* RequestOutput() = 0; + virtual TranscoderInputStream* RequestOutput() = 0; // The status of request transcoding virtual ::google::protobuf::util::Status RequestStatus() = 0; diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc index 2dfcf511a17..f42b6d23108 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc @@ -52,29 +52,23 @@ class TranscoderImpl : public Transcoder { std::unique_ptr response_translator) : request_translator_(std::move(request_translator)), response_translator_(std::move(response_translator)), - request_zero_copy_stream_( - request_translator_->Output().CreateZeroCopyInputStream()), - response_zero_copy_stream_( - response_translator_->CreateZeroCopyInputStream()) {} + request_stream_(request_translator_->Output().CreateInputStream()), + response_stream_(response_translator_->CreateInputStream()) {} // Transcoder implementation - pbio::ZeroCopyInputStream* RequestOutput() { - return request_zero_copy_stream_.get(); - } + TranscoderInputStream* RequestOutput() { return request_stream_.get(); } pbutil::Status RequestStatus() { return request_translator_->Output().Status(); } - pbio::ZeroCopyInputStream* ResponseOutput() { - return response_zero_copy_stream_.get(); - } + pbio::ZeroCopyInputStream* ResponseOutput() { return response_stream_.get(); } pbutil::Status ResponseStatus() { return response_translator_->Status(); } private: std::unique_ptr request_translator_; std::unique_ptr response_translator_; - std::unique_ptr request_zero_copy_stream_; - std::unique_ptr response_zero_copy_stream_; + std::unique_ptr request_stream_; + std::unique_ptr response_stream_; }; // Converts MethodCallInfo into a RequestInfo structure needed by the @@ -132,7 +126,7 @@ TranscoderFactory::TranscoderFactory(const ::google::api::Service& service) pbutil::Status TranscoderFactory::Create( const MethodCallInfo& call_info, pbio::ZeroCopyInputStream* request_input, - pbio::ZeroCopyInputStream* response_input, + TranscoderInputStream* response_input, std::unique_ptr* transcoder) { // Convert MethodCallInfo into RequestInfo RequestInfo request_info; diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h index b7c10865025..d80c93c7f86 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h @@ -19,6 +19,7 @@ #include "contrib/endpoints/include/api_manager/method_call_info.h" #include "contrib/endpoints/src/grpc/transcoding/transcoder.h" +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "contrib/endpoints/src/grpc/transcoding/type_helper.h" #include "google/api/service.pb.h" #include "google/protobuf/io/zero_copy_stream.h" @@ -71,7 +72,7 @@ class TranscoderFactory { ::google::protobuf::util::Status Create( const MethodCallInfo& call_info, ::google::protobuf::io::ZeroCopyInputStream* request_input, - ::google::protobuf::io::ZeroCopyInputStream* response_input, + TranscoderInputStream* response_input, std::unique_ptr* transcoder); private: diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h b/contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h new file mode 100644 index 00000000000..240067d600b --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h @@ -0,0 +1,35 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_TRANSCODER_INPUT_STREAM_H_ +#define GRPC_TRANSCODING_TRANSCODER_INPUT_STREAM_H_ + +#include "google/protobuf/io/zero_copy_stream.h" + +namespace google { +namespace api_manager { +namespace transcoding { + +class TranscoderInputStream + : public virtual google::protobuf::io::ZeroCopyInputStream { + public: + // returns the number of bytes available to read at the moment. + virtual int64_t BytesAvailable() const = 0; +}; + +} // namespace transcoding +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc b/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc index eb2a38eaf5d..a2d09b9a642 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc @@ -90,6 +90,10 @@ class TestMethodInfo : public MethodInfo { return dummy; }; + const std::vector> &metric_cost_vector() const { + return metric_cost_vector_; + } + // Methods that the Transcoder does use const std::string &request_type_url() const { return request_type_url_; } bool request_streaming() const { return request_streaming_; } @@ -104,6 +108,7 @@ class TestMethodInfo : public MethodInfo { bool response_streaming_; std::string body_field_path_; std::string empty_; + std::vector> metric_cost_vector_; }; class TranscoderTest : public ::testing::Test { @@ -139,7 +144,7 @@ class TranscoderTest : public ::testing::Test { } pbutil::Status Build(pbio::ZeroCopyInputStream *request_input, - pbio::ZeroCopyInputStream *response_input, + TranscoderInputStream *response_input, std::unique_ptr *transcoder) { MethodCallInfo call_info; call_info.method_info = method_info_.get(); diff --git a/src/envoy/BUILD b/src/envoy/BUILD index 1c75af50a75..e69de29bb2d 100644 --- a/src/envoy/BUILD +++ b/src/envoy/BUILD @@ -1,34 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -# -cc_test( - name = "envoy_test", - data = [ - "@envoy_git//:envoy-testdata", - ], - copts = [ - "-include ./external/envoy_git/test/precompiled/precompiled_test.h", - ], - deps = [ - "@envoy_git//:envoy-test-lib", - "//external:googletest_main", - ], - args = [ - #TODO: Make all test pass - "--gtest_filter=RouterTest.*", - ], - linkstatic=1, -) diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 132bc364a1d..45c69b7f2d5 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -15,7 +15,6 @@ ################################################################################ # - load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") load("//src/envoy/mixer:proxy_docker.bzl", "proxy_docker_build") load("@protobuf_git//:protobuf.bzl", "cc_proto_library") @@ -31,7 +30,8 @@ cc_proto_library( cc_library( name = "filter_lib", srcs = [ - "forward_attribute_filter.cc", + "config.cc", + "config.h", "http_control.cc", "http_control.h", "http_filter.cc", @@ -41,7 +41,7 @@ cc_library( deps = [ ":string_map_proto", "//external:mixer_client_lib", - "@envoy_git//:envoy-common", + "@envoy//source/exe:envoy_common_lib", ], alwayslink = 1, ) @@ -49,9 +49,11 @@ cc_library( cc_binary( name = "envoy", linkstatic = 1, + linkopts = ["-lrt"], + visibility = [":__subpackages__"], deps = [ ":filter_lib", - "@envoy_git//:envoy-main", + "@envoy//source/exe:envoy_main_lib", ], ) @@ -81,10 +83,6 @@ pkg_tar( ) proxy_docker_build( - images = [ - {"name": "proxy", "base": "@docker_ubuntu//:xenial"}, - {"name": "proxy_debug", "base": "@ubuntu_xenial_debug//file"}, - ], entrypoint = [ "/usr/local/bin/start_envoy", "-e", @@ -94,6 +92,16 @@ proxy_docker_build( "-t", "/etc/opt/proxy/envoy.conf.template", ], + images = [ + { + "name": "proxy", + "base": "@docker_ubuntu//:xenial", + }, + { + "name": "proxy_debug", + "base": "@ubuntu_xenial_debug//file", + }, + ], ports = ["9090"], repository = "istio", tags = ["manual"], diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index d7d39f30e03..48229a3631e 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -3,7 +3,7 @@ This Proxy will use Envoy and talk to Mixer server. ## Build Mixer server -* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: +* Follow https://github.com/istio/mixer/blob/master/doc/dev/development.md to set up environment, and build via: ``` cd $(ISTIO)/mixer @@ -46,54 +46,98 @@ This Proxy will use Envoy and talk to Mixer server. * Then issue HTTP request to proxy. ``` + # request to server-side proxy curl http://localhost:9090/echo -d "hello world" + # request to client-side proxy that gets sent to server-side proxy + curl http://localhost:7070/echo -d "hello world" ``` -## How to configurate HTTP filters - -This module has two HTTP filters: -1. mixer filter: intercept all HTTP requests, call the mixer. -2. forward_attribute filter: Forward attributes to the upstream istio/proxy. - -### *mixer* filter: +## How to configurate HTTP Mixer filters This filter will intercept all HTTP requests and call Mixer. Here is its config: ``` "filters": [ - "type": "both", + "type": "decoder", "name": "mixer", "config": { "mixer_server": "${MIXER_SERVER}", - "attributes" : { + "mixer_attributes" : { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2", + "quota.name": "RequestCount" + }, + "forward_attributes" : { "attribute_name1": "attribute_value1", "attribute_name2": "attribute_value2" - } + }, + "quota_name": "RequestCount", + "quota_amount": "1", + "check_cache_expiration_in_seconds": "600", + "check_cache_keys": [ + "request.host", + "request.path", + "origin.user" + ] } ``` Notes: * mixer_server is required -* attributes: these attributes will be send to the mixer - -### *forward_attribute* HTTP filter: +* mixer_attributes: these attributes will be sent to the mixer in both Check and Report calls. +* forward_attributes: these attributes will be forwarded to the upstream istio/proxy. It will send them to mixer in Check and Report calls. +* quota_name, quota_amount are used for making quota call. quota_amount defaults to 1. +* check_cache_keys is to cache check calls. If missing or empty, check calls are not cached. -This filer will forward attributes to the upstream istio/proxy. +## HTTP Route opaque config +By default, the mixer filter only forwards attributes and does not call mixer server. This behavior can be changed per HTTP route by supplying an opaque config: ``` - "filters": [ - "type": "decoder", - "name": "forward_attribute", - "config": { - "attributes": { - "attribute_name1": "attribute_value1", - "attribute_name2": "attribute_value2" - } - } + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service1", + "opaque_config": { + "mixer_control": "on", + "mixer_forward": "off" + } + } ``` -Notes: -* attributes: these attributes will be forwarded to the upstream istio/proxy. +This route opaque config reverts the behavior by sending requests to mixer server but not forwarding any attributes. +## How to enable quota (rate limiting) + +Quota (rate limiting) is enforced by the mixer. Mixer needs to be configured with Quota in its global config and service config. Its quota config will have +"quota name", its limit within a window. If "Quota" is added but param is missing, the default config is: quota name is "RequestCount", the limit is 10 with 1 second window. Essentially, it is imposing 10 qps rate limiting. + +Mixer client can be configured to make Quota call for all requests. If "quota_name" is specified in the mixer filter config, mixer client will call Quota with the specified quota name. If "quota_amount" is specified, it will call with that amount, otherwise the used amount is 1. + + +## How to pass some attributes from client proxy to mixer. + +Usually client proxy is not configured to call mixer (it can be enabled in the route opaque_config). Client proxy can pass some attributes to mixer by using "forward_attributes" field. Its attributes will be sent to the upstream proxy (the server proxy). If the server proxy is calling mixer, these attributes will be sent to the mixer. + + +## How to enable cache for Check calls + +Check calls can be cached. By default, it is not enabled. It can be enabled by supplying non-empty "check_cache_keys" string list in the mixer filter config. Only these attributes in the Check request, their keys and values, are used to calculate the key for the cache lookup. If it is a cache hit, the cached response will be used. +The cached response will be expired in 5 minutes by default. It can be overrided by supplying "check_cache_expiration_in_seconds" in the mixer filter config. The Check response from the mixer has an expiration field. If it is filled, it will be used. By design, the mixer will control the cache expiration time. + +Following is a sample mixer filter config to enable the Check call cache: +``` + "check_cache_expiration_in_seconds": "600", + "check_cache_keys": [ + "request.host", + "request.path", + "source.labels", + "request.headers/:method", + "origin.user" + ] +``` +For the string map attributes in the above example: +1) "request.headers" attribute is a string map, "request.headers/:method" cache key means only its ":method" key and value are used for cache key. +2) "source.labels" attribute is a string map, "source.labels" cache key means all key value pairs for the string map will be used. diff --git a/src/envoy/mixer/config.cc b/src/envoy/mixer/config.cc new file mode 100644 index 00000000000..8e06b1e9c2b --- /dev/null +++ b/src/envoy/mixer/config.cc @@ -0,0 +1,97 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/config.h" + +using ::istio::mixer_client::Attributes; + +namespace Http { +namespace Mixer { +namespace { + +// The Json object name for mixer-server. +const std::string kMixerServer("mixer_server"); + +// The Json object name for static attributes. +const std::string kMixerAttributes("mixer_attributes"); + +// The Json object name to specify attributes which will be forwarded +// to the upstream istio proxy. +const std::string kForwardAttributes("forward_attributes"); + +// The Json object name for quota name and amount. +const std::string kQuotaName("quota_name"); +const std::string kQuotaAmount("quota_amount"); + +// The Json object name for check cache keys. +const std::string kCheckCacheKeys("check_cache_keys"); +const std::string kCheckCacheExpiration("check_cache_expiration_in_seconds"); + +void ReadString(const Json::Object& json, const std::string& name, + std::string* value) { + if (json.hasObject(name)) { + *value = json.getString(name); + } +} + +void ReadStringMap(const Json::Object& json, const std::string& name, + std::map* map) { + if (json.hasObject(name)) { + json.getObject(name)->iterate( + [map](const std::string& key, const Json::Object& obj) -> bool { + (*map)[key] = obj.asString(); + return true; + }); + } +} + +void ReadStringVector(const Json::Object& json, const std::string& name, + std::vector* value) { + if (json.hasObject(name)) { + auto v = json.getStringArray(name); + value->swap(v); + } +} + +} // namespace + +void MixerConfig::Load(const Json::Object& json) { + ReadString(json, kMixerServer, &mixer_server); + + ReadStringMap(json, kMixerAttributes, &mixer_attributes); + ReadStringMap(json, kForwardAttributes, &forward_attributes); + + ReadString(json, kQuotaName, "a_name); + ReadString(json, kQuotaAmount, "a_amount); + + ReadStringVector(json, kCheckCacheKeys, &check_cache_keys); + ReadString(json, kCheckCacheExpiration, &check_cache_expiration); +} + +void MixerConfig::ExtractQuotaAttributes(Attributes* attr) const { + if (!quota_name.empty()) { + attr->attributes[Attributes::kQuotaName] = + Attributes::StringValue(quota_name); + + int64_t amount = 1; // default amount to 1. + if (!quota_amount.empty()) { + amount = std::stoi(quota_amount); + } + attr->attributes[Attributes::kQuotaAmount] = Attributes::Int64Value(amount); + } +} + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/config.h b/src/envoy/mixer/config.h new file mode 100644 index 00000000000..fc08f33c439 --- /dev/null +++ b/src/envoy/mixer/config.h @@ -0,0 +1,54 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "envoy/json/json_object.h" +#include "include/attribute.h" + +namespace Http { +namespace Mixer { + +// A config for mixer filter +struct MixerConfig { + // the mixer server address + std::string mixer_server; + + // These static attributes will be send to mixer in both + // Check and Report. + std::map mixer_attributes; + + // These attributes will be forwarded to upstream. + std::map forward_attributes; + + // Quota attributes. + std::string quota_name; + std::string quota_amount; + + // The attribute names for check cache. + std::vector check_cache_keys; + std::string check_cache_expiration; + + // Load the config from envoy config. + void Load(const Json::Object& json); + + // Extract quota attributes. + void ExtractQuotaAttributes(::istio::mixer_client::Attributes* attr) const; +}; + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template index c5377632388..c7574734ff3 100644 --- a/src/envoy/mixer/envoy.conf.template +++ b/src/envoy/mixer/envoy.conf.template @@ -1,7 +1,7 @@ { "listeners": [ { - "port": ${PORT}, + "address": "tcp://0.0.0.0:${PORT}", "bind_to_port": true, "filters": [ { @@ -19,7 +19,11 @@ { "timeout_ms": 0, "prefix": "/", - "cluster": "service1" + "cluster": "service1", + "opaque_config": { + "mixer_control": "on", + "mixer_forward": "off" + } } ] } @@ -32,14 +36,21 @@ ], "filters": [ { - "type": "both", + "type": "decoder", "name": "mixer", "config": { "mixer_server": "${MIXER_SERVER}", - "attributes": { + "mixer_attributes": { "target.uid": "POD222", "target.namespace": "XYZ222" - } + }, + "quota_name": "RequestCount", + "quota_amount": "1", + "check_cache_keys": [ + "request.host", + "request.path", + "origin.user" + ] } }, { @@ -53,7 +64,7 @@ ] }, { - "port": 7070, + "address": "tcp://0.0.0.0:7070", "bind_to_port": true, "filters": [ { @@ -85,9 +96,10 @@ "filters": [ { "type": "decoder", - "name": "forward_attribute", + "name": "mixer", "config": { - "attributes": { + "mixer_server": "${MIXER_SERVER}", + "forward_attributes": { "source.uid": "POD11", "source.namespace": "XYZ11" } @@ -106,7 +118,7 @@ ], "admin": { "access_log_path": "/dev/stdout", - "port": 9001 + "address": "tcp://0.0.0.0:9001" }, "cluster_manager": { "clusters": [ diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 3b97d20a35f..61aa3285a62 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -23,8 +23,12 @@ #include "src/envoy/mixer/utils.h" using ::google::protobuf::util::Status; +using ::istio::mixer_client::CheckOptions; using ::istio::mixer_client::Attributes; using ::istio::mixer_client::DoneFunc; +using ::istio::mixer_client::MixerClientOptions; +using ::istio::mixer_client::ReportOptions; +using ::istio::mixer_client::QuotaOptions; namespace Http { namespace Mixer { @@ -40,11 +44,31 @@ const std::string kRequestSize = "request.size"; const std::string kRequestTime = "request.time"; const std::string kResponseHeaders = "response.headers"; -const std::string kResponseHttpCode = "response.http.code"; -const std::string kResponseLatency = "response.latency"; +const std::string kResponseCode = "response.code"; +const std::string kResponseDuration = "response.duration"; const std::string kResponseSize = "response.size"; const std::string kResponseTime = "response.time"; +// Check cache size: 10000 cache entries. +const int kCheckCacheEntries = 10000; +// Default check cache expired in 5 minutes. +const int kCheckCacheExpirationInSeconds = 300; + +CheckOptions GetCheckOptions(const MixerConfig& config) { + int expiration = kCheckCacheExpirationInSeconds; + if (!config.check_cache_expiration.empty()) { + expiration = std::stoi(config.check_cache_expiration); + } + + // Remove expired items from cache 1 second later. + CheckOptions options(kCheckCacheEntries, expiration * 1000, + (expiration + 1) * 1000); + + options.cache_keys = config.check_cache_keys; + + return options; +} + void SetStringAttribute(const std::string& name, const std::string& value, Attributes* attr) { if (!value.empty()) { @@ -94,28 +118,27 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, attr->attributes[kResponseSize] = Attributes::Int64Value(info.bytesSent()); } - if (info.duration().count() > 0) { - attr->attributes[kResponseLatency] = Attributes::DurationValue( - std::chrono::duration_cast(info.duration())); - } + attr->attributes[kResponseDuration] = Attributes::DurationValue( + std::chrono::duration_cast(info.duration())); if (info.responseCode().valid()) { - attr->attributes[kResponseHttpCode] = + attr->attributes[kResponseCode] = Attributes::Int64Value(info.responseCode().value()); } else { - attr->attributes[kResponseHttpCode] = - Attributes::Int64Value(check_status_code); + attr->attributes[kResponseCode] = Attributes::Int64Value(check_status_code); } } } // namespace -HttpControl::HttpControl(const std::string& mixer_server, - std::map&& attributes) - : config_attributes_(std::move(attributes)) { - ::istio::mixer_client::MixerClientOptions options; - options.mixer_server = mixer_server; +HttpControl::HttpControl(const MixerConfig& mixer_config) + : mixer_config_(mixer_config) { + MixerClientOptions options(GetCheckOptions(mixer_config), ReportOptions(), + QuotaOptions()); + options.mixer_server = mixer_config_.mixer_server; mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + + mixer_config_.ExtractQuotaAttributes("a_attributes_); } void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { @@ -133,7 +156,7 @@ void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { FillRequestHeaderAttributes(header_map, attr); - for (const auto& attribute : config_attributes_) { + for (const auto& attribute : mixer_config_.mixer_attributes) { SetStringAttribute(attribute.first, attribute.second, attr); } } @@ -143,7 +166,18 @@ void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, FillCheckAttributes(headers, &request_data->attributes); SetStringAttribute(kOriginUser, origin_user, &request_data->attributes); log().debug("Send Check: {}", request_data->attributes.DebugString()); - mixer_client_->Check(request_data->attributes, on_done); + + auto check_on_done = [this, on_done](const Status& status) { + if (status.ok()) { + if (!quota_attributes_.attributes.empty()) { + log().debug("Send Quota: {}", quota_attributes_.DebugString()); + mixer_client_->Quota(quota_attributes_, on_done); + return; // Not to call on_done again. + } + } + on_done(status); + }; + mixer_client_->Check(request_data->attributes, check_on_done); } void HttpControl::Report(HttpRequestDataPtr request_data, diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index e9ddc734f45..c473facef3c 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -21,6 +21,7 @@ #include "common/http/headers.h" #include "envoy/http/access_log.h" #include "include/client.h" +#include "src/envoy/mixer/config.h" namespace Http { namespace Mixer { @@ -37,8 +38,7 @@ typedef std::shared_ptr HttpRequestDataPtr; class HttpControl final : public Logger::Loggable { public: // The constructor. - HttpControl(const std::string& mixer_server, - std::map&& attributes); + HttpControl(const MixerConfig& mixer_config); // Make mixer check call. void Check(HttpRequestDataPtr request_data, HeaderMap& headers, @@ -56,8 +56,10 @@ class HttpControl final : public Logger::Loggable { // The mixer client std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; - // The attributes read from the config file. - std::map config_attributes_; + // The mixer config + const MixerConfig& mixer_config_; + // Quota attributes; extracted from envoy filter config. + ::istio::mixer_client::Attributes quota_attributes_; }; } // namespace Mixer diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index c99329aea4f..9f8e4ffb75c 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -15,12 +15,14 @@ #include "precompiled/precompiled.h" +#include "common/common/base64.h" #include "common/common/logger.h" #include "common/http/headers.h" #include "common/http/utility.h" #include "envoy/server/instance.h" #include "envoy/ssl/connection.h" #include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/config.h" #include "src/envoy/mixer/http_control.h" #include "src/envoy/mixer/utils.h" @@ -32,11 +34,11 @@ namespace Http { namespace Mixer { namespace { -// The Json object name for mixer-server. -const std::string kJsonNameMixerServer("mixer_server"); +// Switch to turn off attribute forwarding +const std::string kJsonNameForwardSwitch("mixer_forward"); -// The Json object name for static attributes. -const std::string kJsonNameMixerAttributes("attributes"); +// Switch to turn off mixer check/report/quota +const std::string kJsonNameMixerSwitch("mixer_control"); // Convert Status::code to HTTP code int HttpCode(int code) { @@ -88,50 +90,90 @@ class Config : public Logger::Loggable { private: std::shared_ptr http_control_; Upstream::ClusterManager& cm_; + std::string forward_attributes_; + MixerConfig mixer_config_; public: Config(const Json::Object& config, Server::Instance& server) : cm_(server.clusterManager()) { - std::string mixer_server; - if (config.hasObject(kJsonNameMixerServer)) { - mixer_server = config.getString(kJsonNameMixerServer); - } else { + mixer_config_.Load(config); + if (mixer_config_.mixer_server.empty()) { log().error( "mixer_server is required but not specified in the config: {}", __func__); + } else { + log().debug("Called Mixer::Config constructor with mixer_server: ", + mixer_config_.mixer_server); } - std::map attributes = - Utils::ExtractStringMap(config, kJsonNameMixerAttributes); + if (!mixer_config_.forward_attributes.empty()) { + std::string serialized_str = + Utils::SerializeStringMap(mixer_config_.forward_attributes); + forward_attributes_ = + Base64::encode(serialized_str.c_str(), serialized_str.size()); + log().debug("Mixer forward attributes set: ", serialized_str); + } - http_control_ = - std::make_shared(mixer_server, std::move(attributes)); - log().debug("Called Mixer::Config constructor with mixer_server: ", - mixer_server); + http_control_ = std::make_shared(mixer_config_); } std::shared_ptr& http_control() { return http_control_; } + const std::string& forward_attributes() const { return forward_attributes_; } }; typedef std::shared_ptr ConfigPtr; -class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { +class Instance : public Http::StreamDecoderFilter, + public Http::AccessLog::Instance { private: std::shared_ptr http_control_; + ConfigPtr config_; std::shared_ptr request_data_; enum State { NotStarted, Calling, Complete, Responded }; State state_; StreamDecoderFilterCallbacks* decoder_callbacks_; - StreamEncoderFilterCallbacks* encoder_callbacks_; bool initiating_call_; int check_status_code_; + bool mixer_disabled_; + + // mixer control switch (off by default) + bool mixer_disabled() { + auto route = decoder_callbacks_->route(); + if (route != nullptr) { + auto entry = route->routeEntry(); + if (entry != nullptr) { + auto key = entry->opaqueConfig().find(kJsonNameMixerSwitch); + if (key != entry->opaqueConfig().end() && key->second == "on") { + return false; + } + } + } + return true; + } + + // attribute forward switch (on by default) + bool forward_disabled() { + auto route = decoder_callbacks_->route(); + if (route != nullptr) { + auto entry = route->routeEntry(); + if (entry != nullptr) { + auto key = entry->opaqueConfig().find(kJsonNameForwardSwitch); + if (key != entry->opaqueConfig().end() && key->second == "off") { + return true; + } + } + } + return false; + } + public: Instance(ConfigPtr config) : http_control_(config->http_control()), + config_(config), state_(NotStarted), initiating_call_(false), check_status_code_(HttpCode(StatusCode::UNKNOWN)) { @@ -149,6 +191,17 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { FilterHeadersStatus decodeHeaders(HeaderMap& headers, bool end_stream) override { Log().debug("Called Mixer::Instance : {}", __func__); + + if (!config_->forward_attributes().empty() && !forward_disabled()) { + headers.addStatic(Utils::kIstioAttributeHeader, + config_->forward_attributes()); + } + + mixer_disabled_ = mixer_disabled(); + if (mixer_disabled_) { + return FilterHeadersStatus::Continue; + } + state_ = Calling; initiating_call_ = true; request_data_ = std::make_shared(); @@ -174,6 +227,10 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override { + if (mixer_disabled_) { + return FilterDataStatus::Continue; + } + Log().debug("Called Mixer::Instance : {} ({}, {})", __func__, data.length(), end_stream); if (state_ == Calling) { @@ -183,6 +240,10 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { } FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + if (mixer_disabled_) { + return FilterTrailersStatus::Continue; + } + Log().debug("Called Mixer::Instance : {}", __func__); if (state_ == Calling) { return FilterTrailersStatus::StopIteration; @@ -197,6 +258,7 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { decoder_callbacks_->addResetStreamCallback( [this]() { state_ = Responded; }); } + void completeCheck(const Status& status) { Log().debug("Called Mixer::Instance : check complete {}", status.ToString()); @@ -207,32 +269,13 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { status.ToString()); return; } + state_ = Complete; if (!initiating_call_) { decoder_callbacks_->continueDecoding(); } } - virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called Mixer::Instance : {}", __func__); - return FilterHeadersStatus::Continue; - } - virtual FilterDataStatus encodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called Mixer::Instance : {}", __func__); - return FilterDataStatus::Continue; - } - virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - Log().debug("Called Mixer::Instance : {}", __func__); - return FilterTrailersStatus::Continue; - } - virtual void setEncoderFilterCallbacks( - StreamEncoderFilterCallbacks& callbacks) override { - Log().debug("Called Mixer::Instance : {}", __func__); - encoder_callbacks_ = &callbacks; - } - virtual void log(const HeaderMap* request_headers, const HeaderMap* response_headers, const AccessLog::RequestInfo& request_info) override { @@ -267,7 +310,7 @@ class MixerConfig : public HttpFilterConfigFactory { HttpFilterFactoryCb tryCreateFilterFactory( HttpFilterType type, const std::string& name, const Json::Object& config, const std::string&, Server::Instance& server) override { - if (type != HttpFilterType::Both || name != "mixer") { + if (type != HttpFilterType::Decoder || name != "mixer") { return nullptr; } @@ -277,8 +320,10 @@ class MixerConfig : public HttpFilterConfigFactory { [mixer_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { std::shared_ptr instance( new Http::Mixer::Instance(mixer_config)); - callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); - callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); + callbacks.addStreamDecoderFilter( + Http::StreamDecoderFilterSharedPtr(instance)); + callbacks.addAccessLogHandler( + Http::AccessLog::InstanceSharedPtr(instance)); }; } }; @@ -286,4 +331,4 @@ class MixerConfig : public HttpFilterConfigFactory { static RegisterHttpFilterConfigFactory register_; } // namespace Configuration -} // namespace server +} // namespace Server diff --git a/src/envoy/mixer/integration_test/BUILD b/src/envoy/mixer/integration_test/BUILD new file mode 100644 index 00000000000..a6ff4916db5 --- /dev/null +++ b/src/envoy/mixer/integration_test/BUILD @@ -0,0 +1,64 @@ +# Copyright 2017 Istio Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load(":test_suite.bzl", "go_test_suite") + +go_library( + name = "go_default_library", + srcs = [ + "attributes.go", + "envoy.go", + "envoy_conf.go", + "http_client.go", + "http_server.go", + "mixer_server.go", + "setup.go", + ], + deps = [ + "@com_github_gogo_protobuf//types:go_default_library", + "@com_github_golang_glog//:go_default_library", + "@com_github_golang_protobuf//proto:go_default_library", + "@com_github_googleapis_googleapis//:google/rpc", + "@com_github_istio_api//:mixer/v1", + "@com_github_istio_mixer//pkg/api:go_default_library", + "@com_github_istio_mixer//pkg/attribute:go_default_library", + "@com_github_istio_mixer//pkg/pool:go_default_library", + "@com_github_istio_mixer//pkg/tracing:go_default_library", + "@org_golang_google_grpc//:go_default_library", + ], +) + +go_test_suite( + data = [ + "//src/envoy/mixer:envoy", + ], + library = ":go_default_library", + tags = [ + # Use fixed ports, not in sanbbox, have to be run exclusively. + "exclusive", + # shared memory path /envoy_shared_memory_0 used by Envoy + # hot start is not working in sandbox mode. + "local", + ], + tests = [ + "check_cache_test.go", + "check_report_test.go", + "failed_request_test.go", + "quota_test.go", + ], +) diff --git a/src/envoy/mixer/integration_test/attributes.go b/src/envoy/mixer/integration_test/attributes.go new file mode 100644 index 00000000000..6aecd20d8bb --- /dev/null +++ b/src/envoy/mixer/integration_test/attributes.go @@ -0,0 +1,128 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "fmt" + "reflect" + + "istio.io/mixer/pkg/attribute" +) + +func verifyStringMap(actual map[string]string, expected map[string]interface{}) error { + for k, v := range expected { + vstring := v.(string) + // "-" make sure the key does not exist. + if vstring == "-" { + if _, ok := actual[k]; ok { + return fmt.Errorf("key %+v is NOT expected", k) + } + } else { + if val, ok := actual[k]; ok { + // "*" only check key exist + if val != vstring && vstring != "*" { + return fmt.Errorf("key %+v value doesn't match. Actual %+v, expected %+v", + k, val, vstring) + } + } else { + return fmt.Errorf("key %+v is expected", k) + } + } + } + return nil +} + +// Attributes verification rules: +// +// 1) If value is *, key must exist, but value is not checked. +// 1) If value is -, key must NOT exist. +// 3) At top level attributes, not inside StringMap, all keys must +// be listed. Extra keys are NOT allowed +// 3) Inside StringMap, not need to list all keys. Extra keys are allowed +// +// Attributes provided from envoy config +// * source.id and source.namespace are forwarded from client proxy +// * target.id and target.namespace are from server proxy +// +// HTTP header "x-istio-attributes" is used to forward attributes between +// proxy. It should be removed before calling mixer and backend. +// +func Verify(b *attribute.MutableBag, json_results string) error { + var r map[string]interface{} + if err := json.Unmarshal([]byte(json_results), &r); err != nil { + return fmt.Errorf("unable to decode json %v", err) + } + + all_keys := make(map[string]bool) + for _, k := range b.Names() { + all_keys[k] = true + } + + for k, v := range r { + switch vv := v.(type) { + case string: + // "*" means only checking key. + if vv == "*" { + if _, ok := b.Get(k); !ok { + return fmt.Errorf("attribute %+v is expected", k) + } + } else { + if val, ok := b.Get(k); ok { + if val.(string) != v.(string) { + return fmt.Errorf("attribute %+v value doesn't match. Actual %+v, expected %+v", + k, val.(string), v.(string)) + } + } else { + return fmt.Errorf("attribute %+v is expected", k) + } + } + case float64: + // Json converts all integers to float64, + // Our tests only verify size related attributes which are int64 type + if val, ok := b.Get(k); ok { + vint64 := int64(vv) + if val.(int64) != vint64 { + return fmt.Errorf("attribute %+v value doesn't match. Actual %+v, expected %+v", + k, val.(int64), vint64) + } + } else { + return fmt.Errorf("attribute %+v is expected", k) + } + case map[string]interface{}: + if val, ok := b.Get(k); ok { + if err := verifyStringMap(val.(map[string]string), v.(map[string]interface{})); err != nil { + return fmt.Errorf("attribute %+v StringMap doesn't match: %+v", k, err) + } + } else { + return fmt.Errorf("attribute %+v is expected", k) + } + default: + return fmt.Errorf("attribute %+v is of a type %+v that I don't know how to handle ", + k, reflect.TypeOf(v)) + } + delete(all_keys, k) + + } + + if len(all_keys) > 0 { + var s string + for k, _ := range all_keys { + s += k + ", " + } + return fmt.Errorf("Following attributes are not expected: %s", s) + } + return nil +} diff --git a/src/envoy/mixer/integration_test/check_cache_test.go b/src/envoy/mixer/integration_test/check_cache_test.go new file mode 100644 index 00000000000..ab79320ae82 --- /dev/null +++ b/src/envoy/mixer/integration_test/check_cache_test.go @@ -0,0 +1,40 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" +) + +func TestCheckCache(t *testing.T) { + s, err := SetUp(t, basicConfig+","+checkCacheConfig) + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + // Issues a GET echo request with 0 size body + tag := "OKGet" + for i := 0; i < 10; i++ { + if _, _, err := HTTPGet(url); err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + // Only the first check is called. + s.VerifyCheckCount(tag, 1) + } +} diff --git a/src/envoy/mixer/integration_test/check_report_test.go b/src/envoy/mixer/integration_test/check_report_test.go new file mode 100644 index 00000000000..9b9708c19b2 --- /dev/null +++ b/src/envoy/mixer/integration_test/check_report_test.go @@ -0,0 +1,154 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" +) + +// Check attributes from a good GET request +const checkAttributesOkGet = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a good GET request +const reportAttributesOkGet = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 0, + "response.duration": "*", + "response.code": 200, + "response.headers": { + "date": "*", + "content-type": "text/plain; charset=utf-8", + "content-length": "0", + ":status": "200", + "server": "envoy" + } +} +` + +// Check attributes from a good POST request +const checkAttributesOkPost = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "POST", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a good POST request +const reportAttributesOkPost = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "POST", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 12, + "response.time": "*", + "response.size": 12, + "response.duration": "*", + "response.code": 200, + "response.headers": { + "date": "*", + "content-type": "text/plain", + "content-length": "12", + ":status": "200", + "server": "envoy" + } +} +` + +func TestCheckReportAttributes(t *testing.T) { + s, err := SetUp(t, basicConfig) + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + // Issues a GET echo request with 0 size body + tag := "OKGet" + if _, _, err := HTTPGet(url); err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + s.VerifyCheck(tag, checkAttributesOkGet) + s.VerifyReport(tag, reportAttributesOkGet) + + // Issues a POST request. + tag = "OKPost" + if _, _, err := HTTPPost(url, "text/plain", "Hello World!"); err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + s.VerifyCheck(tag, checkAttributesOkPost) + s.VerifyReport(tag, reportAttributesOkPost) +} diff --git a/src/envoy/mixer/integration_test/envoy.go b/src/envoy/mixer/integration_test/envoy.go new file mode 100644 index 00000000000..b34fb5c240a --- /dev/null +++ b/src/envoy/mixer/integration_test/envoy.go @@ -0,0 +1,88 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "log" + "os" + "os/exec" + "strings" +) + +func getTestBinRootPath() string { + switch { + // custom path + case os.Getenv("TEST_BIN_ROOT") != "": + return os.Getenv("TEST_BIN_ROOT") + // running under bazel + case os.Getenv("TEST_SRCDIR") != "": + return os.Getenv("TEST_SRCDIR") + "/__main__" + // running with native go + case os.Getenv("GOPATH") != "": + list := strings.Split(os.Getenv("GOPATH"), + string(os.PathListSeparator)) + return list[0] + "/bazel-bin" + default: + return "bazel-bin" + } +} + +type Envoy struct { + cmd *exec.Cmd +} + +// Run command and return the merged output from stderr and stdout, error code +func Run(name string, args ...string) (s string, err error) { + log.Println(">", name, strings.Join(args, " ")) + c := exec.Command(name, args...) + bytes, err := c.CombinedOutput() + s = string(bytes) + for _, line := range strings.Split(s, "\n") { + log.Println(line) + } + if err != nil { + log.Println(err) + } + return +} + +func NewEnvoy(conf string) (*Envoy, error) { + bin_path := getTestBinRootPath() + "/src/envoy/mixer/envoy" + log.Printf("Envoy binary: %v\n", bin_path) + + conf_path := "/tmp/envoy.conf" + log.Printf("Envoy config: in %v\n%v\n", conf_path, conf) + if err := CreateEnvoyConf(conf_path, conf); err != nil { + return nil, err + } + + cmd := exec.Command(bin_path, "-c", conf_path, "-l", "debug") + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + return &Envoy{ + cmd: cmd, + }, nil +} + +func (s *Envoy) Start() error { + return s.cmd.Start() +} + +func (s *Envoy) Stop() error { + log.Printf("Kill Envoy ...\n") + err := s.cmd.Process.Kill() + log.Printf("Kill Envoy ... Done\n") + return err +} diff --git a/src/envoy/mixer/integration_test/envoy_conf.go b/src/envoy/mixer/integration_test/envoy_conf.go new file mode 100644 index 00000000000..166b11543aa --- /dev/null +++ b/src/envoy/mixer/integration_test/envoy_conf.go @@ -0,0 +1,244 @@ +// Copyright 2017 Istio Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package test + +import ( + "fmt" + "os" + "text/template" +) + +const ( + // These ports should match with used envoy.conf + // Default is using one in this folder. + ServerProxyPort = 29090 + ClientProxyPort = 27070 + MixerPort = 29091 + BackendPort = 28080 + AdminPort = 29001 +) + +type ConfParam struct { + ClientPort int + ServerPort int + AdminPort int + MixerServer string + Backend string + ClientConfig string + ServerConfig string +} + +// A basic config +const basicConfig = ` + "mixer_attributes": { + "target.uid": "POD222", + "target.namespace": "XYZ222" + } +` + +// A config with quota +const quotaConfig = ` + "quota_name": "RequestCount", + "quota_amount": "5" +` + +// A config with check cache keys +const checkCacheConfig = ` + "check_cache_keys": [ + "request.host", + "request.path", + "origin.user" + ] +` + +// The default client proxy mixer config +const defaultClientMixerConfig = ` + "forward_attributes": { + "source.uid": "POD11", + "source.namespace": "XYZ11" + } +` + +// The envoy config template +const envoyConfTempl = ` +{ + "listeners": [ + { + "address": "tcp://0.0.0.0:{{.ServerPort}}", + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service1", + "opaque_config": { + "mixer_control": "on", + "mixer_forward": "off" + } + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "decoder", + "name": "mixer", + "config": { + "mixer_server": "{{.MixerServer}}", +{{.ServerConfig}} + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + }, + { + "address": "tcp://0.0.0.0:{{.ClientPort}}", + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service2" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "decoder", + "name": "mixer", + "config": { + "mixer_server": "{{.MixerServer}}", +{{.ClientConfig}} + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + } + ], + "admin": { + "access_log_path": "/dev/stdout", + "address": "tcp://0.0.0.0:{{.AdminPort}}" + }, + "cluster_manager": { + "clusters": [ + { + "name": "service1", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://{{.Backend}}" + } + ] + }, + { + "name": "service2", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:{{.ServerPort}}" + } + ] + } + ] + } +} +` + +func (c *ConfParam) write(path string) error { + tmpl, err := template.New("test").Parse(envoyConfTempl) + if err != nil { + return fmt.Errorf("Failed to parse config template: %v", err) + } + + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("Failed to create file %v: %v", path, err) + } + defer f.Close() + return tmpl.Execute(f, *c) +} + +func getConf() ConfParam { + return ConfParam{ + ClientPort: ClientProxyPort, + ServerPort: ServerProxyPort, + AdminPort: AdminPort, + MixerServer: fmt.Sprintf("localhost:%d", MixerPort), + Backend: fmt.Sprintf("localhost:%d", BackendPort), + ClientConfig: defaultClientMixerConfig, + } +} + +func CreateEnvoyConf(path string, conf string) error { + c := getConf() + c.ServerConfig = conf + return c.write(path) +} diff --git a/src/envoy/mixer/integration_test/failed_request_test.go b/src/envoy/mixer/integration_test/failed_request_test.go new file mode 100644 index 00000000000..011227afa6f --- /dev/null +++ b/src/envoy/mixer/integration_test/failed_request_test.go @@ -0,0 +1,161 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" + + rpc "github.com/googleapis/googleapis/google/rpc" +) + +const ( + mixerAuthFailMessage = "Unauthenticated by mixer." +) + +// Check attributes from a fail GET request from mixer +const checkAttributesMixerFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a fail GET request from mixer +const reportAttributesMixerFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 41, + "response.duration": "*", + "response.code": 401, + "response.headers": { + "date": "*", + "content-type": "text/plain", + "content-length": "41", + ":status": "401", + "server": "envoy" + } +} +` + +// Report attributes from a fail GET request from backend +const reportAttributesBackendFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 25, + "response.duration": "*", + "response.code": 400, + "response.headers": { + "date": "*", + "content-type": "text/plain; charset=utf-8", + "content-length": "25", + ":status": "400", + "server": "envoy" + } +} +` + +func TestFailedRequest(t *testing.T) { + s, err := SetUp(t, basicConfig) + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + tag := "MixerFail" + s.mixer.check.r_status = rpc.Status{ + Code: int32(rpc.UNAUTHENTICATED), + Message: mixerAuthFailMessage, + } + code, resp_body, err := HTTPGet(url) + // Make sure to restore r_status for next request. + s.mixer.check.r_status = rpc.Status{} + if err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + if code != 401 { + t.Errorf("Status code 401 is expected.") + } + if resp_body != "UNAUTHENTICATED:"+mixerAuthFailMessage { + t.Errorf("Error response body is not expected.") + } + s.VerifyCheck(tag, checkAttributesMixerFail) + s.VerifyReport(tag, reportAttributesMixerFail) + + // Issues a failed request caused by backend + tag = "BackendFail" + headers := map[string]string{} + headers[FailHeader] = "Yes" + code, resp_body, err = HTTPGetWithHeaders(url, headers) + if err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + if code != 400 { + t.Errorf("Status code 400 is expected.") + } + if resp_body != FailBody { + t.Errorf("Error response body is not expected.") + } + // Same Check attributes as the first one. + s.VerifyCheck(tag, checkAttributesMixerFail) + s.VerifyReport(tag, reportAttributesBackendFail) +} diff --git a/src/envoy/mixer/integration_test/http_client.go b/src/envoy/mixer/integration_test/http_client.go new file mode 100644 index 00000000000..0212ddf775e --- /dev/null +++ b/src/envoy/mixer/integration_test/http_client.go @@ -0,0 +1,80 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "io/ioutil" + "log" + "net/http" + "net/url" + "strings" +) + +func HTTPGet(url string) (code int, resp_body string, err error) { + log.Println("HTTP GET", url) + client := &http.Client{} + resp, err := client.Get(url) + if err != nil { + log.Println(err) + return 0, "", err + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + resp_body = string(body) + code = resp.StatusCode + log.Println(resp_body) + return code, resp_body, nil +} + +func HTTPPost(url string, content_type string, req_body string) (code int, resp_body string, err error) { + log.Println("HTTP POST", url) + client := &http.Client{} + resp, err := client.Post(url, content_type, strings.NewReader(req_body)) + if err != nil { + log.Println(err) + return 0, "", err + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + resp_body = string(body) + code = resp.StatusCode + log.Println(resp_body) + return code, resp_body, nil +} + +func HTTPGetWithHeaders(l string, headers map[string]string) (code int, resp_body string, err error) { + log.Println("HTTP GET with headers: ", l) + client := &http.Client{} + req := http.Request{} + + req.Header = map[string][]string{} + for k, v := range headers { + req.Header[k] = []string{v} + } + req.Method = http.MethodGet + req.URL, _ = url.Parse(l) + + resp, err := client.Do(&req) + if err != nil { + log.Println(err) + return 0, "", err + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + resp_body = string(body) + code = resp.StatusCode + log.Println(resp_body) + return code, resp_body, nil +} diff --git a/src/envoy/mixer/integration_test/http_server.go b/src/envoy/mixer/integration_test/http_server.go new file mode 100644 index 00000000000..77677286b58 --- /dev/null +++ b/src/envoy/mixer/integration_test/http_server.go @@ -0,0 +1,111 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "strings" + "time" +) + +const ( + FailHeader = "x-istio-backend-fail" + FailBody = "Bad request from backend." +) + +type HttpServer struct { + port uint16 + lis net.Listener +} + +func handler(w http.ResponseWriter, r *http.Request) { + log.Printf("%v %v %v %v\n", r.Method, r.URL, r.Proto, r.RemoteAddr) + for name, headers := range r.Header { + for _, h := range headers { + log.Printf("%v: %v\n", name, h) + } + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Fail if there is such header. + if r.Header.Get(FailHeader) != "" { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(FailBody)) + return + } + + // echo back the Content-Type and Content-Length in the response + for _, k := range []string{"Content-Type", "Content-Length"} { + if v := r.Header.Get(k); v != "" { + w.Header().Set(k, v) + } + } + w.WriteHeader(http.StatusOK) + w.Write(body) +} + +func NewHttpServer(port uint16) (*HttpServer, error) { + log.Printf("Http server listening on port %v\n", port) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + log.Fatal(err) + return nil, err + } + return &HttpServer{ + port: port, + lis: lis, + }, nil +} + +func (s *HttpServer) Start() { + go func() { + http.HandleFunc("/", handler) + http.Serve(s.lis, nil) + }() + + addr := fmt.Sprintf("http://localhost:%s", s.port) + + const maxAttempts = 10 + for i := 0; i < maxAttempts; i++ { + time.Sleep(time.Second) + client := http.Client{} + log.Println("Pinging the server...") + rsp, err := client.Post( + addr+"/echo", "text/plain", strings.NewReader("PING")) + if err == nil && rsp.StatusCode == http.StatusOK { + log.Println("Got a response...") + png, err := ioutil.ReadAll(rsp.Body) + if err == nil && string(png) == "PING" { + log.Println("Server is up and running...") + return + } + } + log.Println("Will wait a second and try again.") + } +} + +func (s *HttpServer) Stop() { + log.Printf("Close HTTP server\n") + s.lis.Close() + log.Printf("Close HTTP server -- Done\n") +} diff --git a/src/envoy/mixer/integration_test/mixer_server.go b/src/envoy/mixer/integration_test/mixer_server.go new file mode 100644 index 00000000000..6e3db085e5d --- /dev/null +++ b/src/envoy/mixer/integration_test/mixer_server.go @@ -0,0 +1,127 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "fmt" + "log" + "net" + + rpc "github.com/googleapis/googleapis/google/rpc" + "google.golang.org/grpc" + + mixerpb "istio.io/api/mixer/v1" + "istio.io/mixer/pkg/api" + "istio.io/mixer/pkg/attribute" + "istio.io/mixer/pkg/pool" + "istio.io/mixer/pkg/tracing" +) + +type Handler struct { + bag *attribute.MutableBag + ch chan int + count int + r_status rpc.Status +} + +func newHandler() *Handler { + return &Handler{ + bag: nil, + ch: make(chan int, 10), // Allow maximum 10 requests + count: 0, + r_status: rpc.Status{}, + } +} + +func (h *Handler) run(bag *attribute.MutableBag) rpc.Status { + h.bag = attribute.CopyBag(bag) + h.ch <- 1 + h.count++ + return h.r_status +} + +type MixerServer struct { + lis net.Listener + gs *grpc.Server + gp *pool.GoroutinePool + s mixerpb.MixerServer + + check *Handler + report *Handler + quota *Handler + quota_request *mixerpb.QuotaRequest +} + +func (ts *MixerServer) Check(ctx context.Context, bag *attribute.MutableBag, + request *mixerpb.CheckRequest, response *mixerpb.CheckResponse) { + response.RequestIndex = request.RequestIndex + response.Result = ts.check.run(bag) +} + +func (ts *MixerServer) Report(ctx context.Context, bag *attribute.MutableBag, + request *mixerpb.ReportRequest, response *mixerpb.ReportResponse) { + response.RequestIndex = request.RequestIndex + response.Result = ts.report.run(bag) +} + +func (ts *MixerServer) Quota(ctx context.Context, bag *attribute.MutableBag, + request *mixerpb.QuotaRequest, response *mixerpb.QuotaResponse) { + response.RequestIndex = request.RequestIndex + ts.quota_request = request + response.Result = ts.quota.run(bag) + response.Amount = 0 +} + +func NewMixerServer(port uint16) (*MixerServer, error) { + log.Printf("Mixer server listening on port %v\n", port) + s := &MixerServer{ + check: newHandler(), + report: newHandler(), + quota: newHandler(), + } + + var err error + s.lis, err = net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + return nil, err + } + + var opts []grpc.ServerOption + opts = append(opts, grpc.MaxConcurrentStreams(32)) + opts = append(opts, grpc.MaxMsgSize(1024*1024)) + s.gs = grpc.NewServer(opts...) + + s.gp = pool.NewGoroutinePool(128, false) + s.gp.AddWorkers(32) + + s.s = api.NewGRPCServer(s, tracing.DisabledTracer(), s.gp) + mixerpb.RegisterMixerServer(s.gs, s.s) + return s, nil +} + +func (s *MixerServer) Start() { + go func() { + _ = s.gs.Serve(s.lis) + log.Printf("Mixer server exited\n") + }() +} + +func (s *MixerServer) Stop() { + log.Printf("Stop Mixer server\n") + s.gs.Stop() + log.Printf("Stop Mixer server -- Done\n") +} diff --git a/src/envoy/mixer/integration_test/quota_test.go b/src/envoy/mixer/integration_test/quota_test.go new file mode 100644 index 00000000000..e99afb8e426 --- /dev/null +++ b/src/envoy/mixer/integration_test/quota_test.go @@ -0,0 +1,64 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" + + rpc "github.com/googleapis/googleapis/google/rpc" +) + +const ( + mixerQuotaFailMessage = "Not enough quota by mixer." +) + +func TestQuotaCall(t *testing.T) { + s, err := SetUp(t, basicConfig+","+quotaConfig) + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + // Issues a GET echo request with 0 size body + tag := "OKGet" + if _, _, err := HTTPGet(url); err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + s.VerifyQuota(tag, "RequestCount", 5) + + // Issues a failed POST request caused by Mixer Quota + tag = "QuotaFail" + s.mixer.quota_request = nil + s.mixer.quota.r_status = rpc.Status{ + Code: int32(rpc.RESOURCE_EXHAUSTED), + Message: mixerQuotaFailMessage, + } + code, resp_body, err := HTTPPost(url, "text/plain", "Hello World!") + // Make sure to restore r_status for next request. + s.mixer.quota.r_status = rpc.Status{} + if err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + if code != 429 { + t.Errorf("Status code 429 is expected.") + } + if resp_body != "RESOURCE_EXHAUSTED:"+mixerQuotaFailMessage { + t.Errorf("Error response body is not expected.") + } + s.VerifyQuota(tag, "RequestCount", 5) +} diff --git a/src/envoy/mixer/integration_test/repositories.bzl b/src/envoy/mixer/integration_test/repositories.bzl new file mode 100644 index 00000000000..7eec527fa39 --- /dev/null +++ b/src/envoy/mixer/integration_test/repositories.bzl @@ -0,0 +1,272 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_repositories", "new_go_repository", "go_repository") +load("@org_pubref_rules_protobuf//protobuf:rules.bzl", "proto_repositories") + +load("@org_pubref_rules_protobuf//gogo:rules.bzl", "gogo_proto_repositories") +load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cpp_proto_repositories") + +def go_istio_api_repositories(use_local=False): + ISTIO_API_BUILD_FILE = """ +# build protos from istio.io/api repo + +package(default_visibility = ["//visibility:public"]) + +load("@io_bazel_rules_go//go:def.bzl", "go_prefix") + +go_prefix("istio.io/api") + +load("@org_pubref_rules_protobuf//gogo:rules.bzl", "gogoslick_proto_library") + +gogoslick_proto_library( + name = "mixer/v1", + importmap = { + "gogoproto/gogo.proto": "github.com/gogo/protobuf/gogoproto", + "google/rpc/status.proto": "github.com/googleapis/googleapis/google/rpc", + "google/protobuf/timestamp.proto": "github.com/gogo/protobuf/types", + "google/protobuf/duration.proto": "github.com/gogo/protobuf/types", + }, + imports = [ + "../../external/com_github_gogo_protobuf", + "../../external/com_github_google_protobuf/src", + "../../external/com_github_googleapis_googleapis", + ], + inputs = [ + "@com_github_google_protobuf//:well_known_protos", + "@com_github_googleapis_googleapis//:status_proto", + "@com_github_gogo_protobuf//gogoproto:go_default_library_protos", + ], + protos = [ + "mixer/v1/attributes.proto", + "mixer/v1/check.proto", + "mixer/v1/quota.proto", + "mixer/v1/report.proto", + "mixer/v1/service.proto", + ], + verbose = 0, + visibility = ["//visibility:public"], + with_grpc = True, + deps = [ + "@com_github_gogo_protobuf//gogoproto:go_default_library", + "@com_github_gogo_protobuf//sortkeys:go_default_library", + "@com_github_gogo_protobuf//types:go_default_library", + "@com_github_googleapis_googleapis//:google/rpc", + ], +) + +DESCRIPTOR_FILE_GROUP = [ + "mixer/v1/config/descriptor/attribute_descriptor.proto", + "mixer/v1/config/descriptor/label_descriptor.proto", + "mixer/v1/config/descriptor/log_entry_descriptor.proto", + "mixer/v1/config/descriptor/metric_descriptor.proto", + "mixer/v1/config/descriptor/monitored_resource_descriptor.proto", + "mixer/v1/config/descriptor/principal_descriptor.proto", + "mixer/v1/config/descriptor/quota_descriptor.proto", + "mixer/v1/config/descriptor/value_type.proto", +] + +gogoslick_proto_library( + name = "mixer/v1/config", + importmap = { + "google/protobuf/struct.proto": "github.com/gogo/protobuf/types", + "mixer/v1/config/descriptor/log_entry_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + "mixer/v1/config/descriptor/metric_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + "mixer/v1/config/descriptor/monitored_resource_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + "mixer/v1/config/descriptor/principal_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + "mixer/v1/config/descriptor/quota_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + }, + imports = [ + "../../external/com_github_google_protobuf/src", + ], + inputs = DESCRIPTOR_FILE_GROUP + [ + "@com_github_google_protobuf//:well_known_protos", + ], + protos = [ + "mixer/v1/config/cfg.proto", + ], + verbose = 0, + visibility = ["//visibility:public"], + with_grpc = False, + deps = [ + ":mixer/v1/config/descriptor", + "@com_github_gogo_protobuf//sortkeys:go_default_library", + "@com_github_gogo_protobuf//types:go_default_library", + "@com_github_googleapis_googleapis//:google/rpc", + ], +) + +gogoslick_proto_library( + name = "mixer/v1/config/descriptor", + importmap = { + "google/protobuf/duration.proto": "github.com/gogo/protobuf/types", + }, + imports = [ + "../../external/com_github_google_protobuf/src", + ], + inputs = [ + "@com_github_google_protobuf//:well_known_protos", + ], + protos = DESCRIPTOR_FILE_GROUP, + verbose = 0, + visibility = ["//visibility:public"], + with_grpc = False, + deps = [ + "@com_github_gogo_protobuf//types:go_default_library", + ], +) +""" + if use_local: + native.new_local_repository( + name = "com_github_istio_api", + build_file_content = ISTIO_API_BUILD_FILE, + path = "../api", + ) + else: + native.new_git_repository( + name = "com_github_istio_api", + build_file_content = ISTIO_API_BUILD_FILE, + commit = "2cb09827d7f09a6e88eac2c2249dcb45c5419f09", # Mar. 14, 2017 (no releases) + remote = "https://github.com/istio/api.git", + ) + +def go_googleapis_repositories(): + GOOGLEAPIS_BUILD_FILE = """ +package(default_visibility = ["//visibility:public"]) + +load("@io_bazel_rules_go//go:def.bzl", "go_prefix") +go_prefix("github.com/googleapis/googleapis") + +load("@org_pubref_rules_protobuf//gogo:rules.bzl", "gogoslick_proto_library") + +gogoslick_proto_library( + name = "google/rpc", + protos = [ + "google/rpc/code.proto", + "google/rpc/error_details.proto", + "google/rpc/status.proto", + ], + importmap = { + "google/protobuf/any.proto": "github.com/gogo/protobuf/types", + "google/protobuf/duration.proto": "github.com/gogo/protobuf/types", + }, + imports = [ + "../../external/com_github_google_protobuf/src", + ], + inputs = [ + "@com_github_google_protobuf//:well_known_protos", + ], + deps = [ + "@com_github_gogo_protobuf//types:go_default_library", + ], + verbose = 0, +) + +load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cc_proto_library") + +cc_proto_library( + name = "cc_status_proto", + protos = [ + "google/rpc/status.proto", + ], + imports = [ + "../../external/com_github_google_protobuf/src", + ], + verbose = 0, +) + +filegroup( + name = "status_proto", + srcs = [ "google/rpc/status.proto" ], +) + +filegroup( + name = "code_proto", + srcs = [ "google/rpc/code.proto" ], +) +""" + native.new_git_repository( + name = "com_github_googleapis_googleapis", + build_file_content = GOOGLEAPIS_BUILD_FILE, + commit = "13ac2436c5e3d568bd0e938f6ed58b77a48aba15", # Oct 21, 2016 (only release pre-dates sha) + remote = "https://github.com/googleapis/googleapis.git", + ) + +def go_mixer_repositories(use_local_api=False): + go_istio_api_repositories(use_local_api) + go_googleapis_repositories() + + go_repositories() + proto_repositories() + + gogo_proto_repositories() + + new_go_repository( + name = "com_github_golang_glog", + commit = "23def4e6c14b4da8ac2ed8007337bc5eb5007998", # Jan 26, 2016 (no releases) + importpath = "github.com/golang/glog", + ) + + new_go_repository( + name = "com_github_ghodss_yaml", + commit = "04f313413ffd65ce25f2541bfd2b2ceec5c0908c", # Dec 6, 2016 (no releases) + importpath = "github.com/ghodss/yaml", + ) + + new_go_repository( + name = "in_gopkg_yaml_v2", + commit = "14227de293ca979cf205cd88769fe71ed96a97e2", # Jan 24, 2017 (no releases) + importpath = "gopkg.in/yaml.v2", + ) + + new_go_repository( + name = "com_github_golang_protobuf", + commit = "8ee79997227bf9b34611aee7946ae64735e6fd93", # Nov 16, 2016 (no releases) + importpath = "github.com/golang/protobuf", + ) + + new_go_repository( + name = "org_golang_google_grpc", + commit = "708a7f9f3283aa2d4f6132d287d78683babe55c8", # Dec 5, 2016 (v1.0.5) + importpath = "google.golang.org/grpc", + ) + + new_go_repository( + name = "com_github_spf13_cobra", + commit = "35136c09d8da66b901337c6e86fd8e88a1a255bd", # Jan 30, 2017 (no releases) + importpath = "github.com/spf13/cobra", + ) + + new_go_repository( + name = "com_github_spf13_pflag", + commit = "9ff6c6923cfffbcd502984b8e0c80539a94968b7", # Jan 30, 2017 (no releases) + importpath = "github.com/spf13/pflag", + ) + + new_go_repository( + name = "com_github_hashicorp_go_multierror", + commit = "ed905158d87462226a13fe39ddf685ea65f1c11f", # Dec 16, 2016 (no releases) + importpath = "github.com/hashicorp/go-multierror", + ) + + new_go_repository( + name = "com_github_hashicorp_errwrap", + commit = "7554cd9344cec97297fa6649b055a8c98c2a1e55", # Oct 27, 2014 (no releases) + importpath = "github.com/hashicorp/errwrap", + ) + + new_go_repository( + name = "com_github_opentracing_opentracing_go", + commit = "0c3154a3c2ce79d3271985848659870599dfb77c", # Sep 26, 2016 (v1.0.0) + importpath = "github.com/opentracing/opentracing-go", + ) + + new_go_repository( + name = "com_github_opentracing_basictracer", + commit = "1b32af207119a14b1b231d451df3ed04a72efebf", # Sep 29, 2016 (no releases) + importpath = "github.com/opentracing/basictracer-go", + ) + + go_repository( + name = "com_github_istio_mixer", + commit = "064001053b51f73adc3a80ff87ef41a15316c300", + importpath = "github.com/istio/mixer", + ) + diff --git a/src/envoy/mixer/integration_test/setup.go b/src/envoy/mixer/integration_test/setup.go new file mode 100644 index 00000000000..52bfdd2f711 --- /dev/null +++ b/src/envoy/mixer/integration_test/setup.go @@ -0,0 +1,93 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "log" + "testing" +) + +type TestSetup struct { + envoy *Envoy + mixer *MixerServer + backend *HttpServer + t *testing.T +} + +func SetUp(t *testing.T, conf string) (s TestSetup, err error) { + s.t = t + s.envoy, err = NewEnvoy(conf) + if err != nil { + log.Printf("unable to create Envoy %v", err) + } else { + s.envoy.Start() + } + + s.mixer, err = NewMixerServer(MixerPort) + if err != nil { + log.Printf("unable to create mixer server %v", err) + } else { + s.mixer.Start() + } + + s.backend, err = NewHttpServer(BackendPort) + if err != nil { + log.Printf("unable to create HTTP server %v", err) + } else { + s.backend.Start() + } + return s, err +} + +func (s *TestSetup) TearDown() { + s.envoy.Stop() + s.mixer.Stop() + s.backend.Stop() +} + +func (s *TestSetup) VerifyCheckCount(tag string, expected int) { + if s.mixer.check.count != expected { + s.t.Fatalf("%s check count doesn't match: %v\n, expected: %+v", + tag, s.mixer.check.count, expected) + } +} + +func (s *TestSetup) VerifyCheck(tag string, result string) { + _ = <-s.mixer.check.ch + if err := Verify(s.mixer.check.bag, result); err != nil { + s.t.Fatalf("Failed to verify %s check: %v\n, Attributes: %+v", + tag, err, s.mixer.check.bag) + } +} + +func (s *TestSetup) VerifyReport(tag string, result string) { + _ = <-s.mixer.report.ch + if err := Verify(s.mixer.report.bag, result); err != nil { + s.t.Fatalf("Failed to verify %s report: %v\n, Attributes: %+v", + tag, err, s.mixer.report.bag) + } +} + +func (s *TestSetup) VerifyQuota(tag string, name string, amount int64) { + _ = <-s.mixer.quota.ch + if s.mixer.quota_request.Quota != name { + s.t.Fatalf("Failed to verify %s quota name: %v, expected: %v\n", + tag, s.mixer.quota_request.Quota, name) + } + if s.mixer.quota_request.Amount != amount { + s.t.Fatalf("Failed to verify %s quota amount: %v, expected: %v\n", + tag, s.mixer.quota_request.Amount, amount) + } +} diff --git a/src/envoy/mixer/integration_test/test_suite.bzl b/src/envoy/mixer/integration_test/test_suite.bzl new file mode 100644 index 00000000000..afa984f4773 --- /dev/null +++ b/src/envoy/mixer/integration_test/test_suite.bzl @@ -0,0 +1,29 @@ +# Copyright 2017 Istio Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +def go_test_suite(tests, library, data, tags, size="small"): + for test in tests: + go_test( + name = test.split(".")[0], + size = size, + srcs = [test], + data = data, + library = library, + tags = tags, + ) diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 473e2650882..4382e71f921 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "92a305961bcea90ed349ffdbb0ea299c6f6bacad" +MIXER_CLIENT = "c5d857e28bfcc53f20f59466b464f99526737545" def mixer_client_repositories(bind=True): native.git_repository( diff --git a/src/envoy/mixer/utils.cc b/src/envoy/mixer/utils.cc index 6dc3a78356d..32b09667208 100644 --- a/src/envoy/mixer/utils.cc +++ b/src/envoy/mixer/utils.cc @@ -21,20 +21,6 @@ namespace Utils { const LowerCaseString kIstioAttributeHeader("x-istio-attributes"); -StringMap ExtractStringMap(const Json::Object& json, const std::string& name) { - StringMap map; - if (json.hasObject(name)) { - Json::ObjectPtr json_obj = json.getObject(name); - Json::Object* raw_obj = json_obj.get(); - json_obj->iterate( - [&map, raw_obj](const std::string& key, const Json::Object&) -> bool { - map[key] = raw_obj->getString(key); - return true; - }); - } - return map; -} - std::string SerializeStringMap(const StringMap& string_map) { ::istio::proxy::mixer::StringMap pb; ::google::protobuf::Map* map_pb = pb.mutable_map(); diff --git a/src/envoy/mixer/utils.h b/src/envoy/mixer/utils.h index 9273a9a2d8b..e94ed91706c 100644 --- a/src/envoy/mixer/utils.h +++ b/src/envoy/mixer/utils.h @@ -29,9 +29,6 @@ extern const LowerCaseString kIstioAttributeHeader; // The string map. typedef std::map StringMap; -// Extracts name/value attributes from a json object. -StringMap ExtractStringMap(const Json::Object& json, const std::string& name); - // Serialize a string map to string. std::string SerializeStringMap(const StringMap& map); diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index dffa32438a0..bad332e188d 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -235,25 +235,6 @@ def lightstep_repositories(bind=True): BUILD = """ load("@protobuf_git//:protobuf.bzl", "cc_proto_library") -genrule( - name = "envoy_carrier_pb", - srcs = ["src/c++11/envoy/envoy_carrier.proto"], - outs = ["lightstep/envoy_carrier.proto"], - cmd = "cp $(SRCS) $@", -) - -cc_proto_library( - name = "envoy_carrier_proto", - srcs = ["lightstep/envoy_carrier.proto"], - include = ".", - deps = [ - "//external:cc_wkt_protos", - ], - protoc = "//external:protoc", - default_runtime = "//external:protobuf", - visibility = ["//visibility:public"], -) - cc_library( name = "lightstep_core", srcs = [ @@ -266,7 +247,7 @@ cc_library( "src/c++11/lightstep/impl.h", "src/c++11/lightstep/options.h", "src/c++11/lightstep/propagation.h", - "src/c++11/lightstep/envoy.h", + "src/c++11/lightstep/carrier.h", "src/c++11/lightstep/span.h", "src/c++11/lightstep/tracer.h", "src/c++11/lightstep/util.h", @@ -275,7 +256,7 @@ cc_library( "src/c++11/mapbox_variant/variant.hpp", ], copts = [ - "-DPACKAGE_VERSION='\\"0.19\\"'", + "-DPACKAGE_VERSION='\\"0.36\\"'", "-Iexternal/lightstep_git/src/c++11/lightstep", "-Iexternal/lightstep_git/src/c++11/mapbox_variant", ], @@ -285,7 +266,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "@lightstep_common_git//:collector_proto", - ":envoy_carrier_proto", + "@lightstep_common_git//:lightstep_carrier_proto", "//external:protobuf", ], )""" @@ -293,16 +274,21 @@ cc_library( COMMON_BUILD = """ load("@protobuf_git//:protobuf.bzl", "cc_proto_library") -genrule( - name = "collector_pb", +cc_proto_library( + name = "collector_proto", srcs = ["collector.proto"], - outs = ["lightstep/collector.proto"], - cmd = "cp $(SRCS) $@", + include = ".", + deps = [ + "//external:cc_wkt_protos", + ], + protoc = "//external:protoc", + default_runtime = "//external:protobuf", + visibility = ["//visibility:public"], ) cc_proto_library( - name = "collector_proto", - srcs = ["lightstep/collector.proto"], + name = "lightstep_carrier_proto", + srcs = ["lightstep_carrier.proto"], include = ".", deps = [ "//external:cc_wkt_protos", @@ -310,19 +296,20 @@ cc_proto_library( protoc = "//external:protoc", default_runtime = "//external:protobuf", visibility = ["//visibility:public"], -)""" +) +""" native.new_git_repository( name = "lightstep_common_git", remote = "https://github.com/lightstep/lightstep-tracer-common.git", - commit = "8d932f7f76cd286691e6179621d0012b0ff1e6aa", + commit = "cbbecd671c1ae1f20ae873c5da688c8c14d04ec3", build_file_content = COMMON_BUILD, ) native.new_git_repository( name = "lightstep_git", remote = "https://github.com/lightstep/lightstep-tracer-cpp.git", - commit = "5a71d623cac17a059041b04fabca4ed86ffff7cc", + commit = "f1dc8f3dfd529350e053fd21273e627f409ae428", # 0.36 build_file_content = BUILD, ) @@ -572,185 +559,14 @@ def envoy_repositories(bind=True): rapidjson_repositories(bind) nghttp2_repositories(bind) ares_repositories(bind) - - BUILD = """ -load("@protobuf_git//:protobuf.bzl", "cc_proto_library") - -exports_files(["source/precompiled/precompiled.h"]) - -package(default_visibility = ["//visibility:public"]) - -genrule( - name = "envoy-ratelimit-proto", - srcs = [ - "source/common/ratelimit/ratelimit.proto", - ], - outs = [ - "source/common/generated/ratelimit.proto", - ], - cmd = "cp $(SRCS) $@", -) - -cc_proto_library( - name = "envoy-ratelimit-pb", - srcs = [ - "source/common/generated/ratelimit.proto", - ], - default_runtime = "//external:protobuf", - protoc = "//external:protoc", - include = "source", -) - -genrule( - name = "envoy-test-proto", - srcs = [ - "test/proto/helloworld.proto", - ], - outs = [ - "test/generated/helloworld.proto", - ], - cmd = "cp $(SRCS) $@", -) - -cc_proto_library( - name = "envoy-test-pb", - srcs = [ - "test/generated/helloworld.proto", - ], - default_runtime = "//external:protobuf", - protoc = "//external:protoc", - include = "test", -) - -genrule( - name = "envoy-version", - srcs = glob([ - ".git/**", - ]), - tools = [ - "tools/gen_git_sha.sh", - ], - outs = [ - "source/common/version_generated.cc", - ], - cmd = "touch $@ && $(location tools/gen_git_sha.sh) $$(dirname $(location tools/gen_git_sha.sh)) $@", - local = 1, -) - -cc_library( - name = "envoy-common", - srcs = glob([ - "source/**/*.cc", - "source/**/*.h", - "include/**/*.h", - ], exclude=["source/exe/main.cc"]) + [ - "source/common/version_generated.cc", - ], - copts = [ - "-I./external/envoy_git/source", - "-include ./external/envoy_git/source/precompiled/precompiled.h", - ], - includes = [ - "include", - ], - linkopts = [ - "-lpthread", - "-lanl", - "-lrt", - ], - linkstatic=1, - alwayslink=1, - deps = [ - ":envoy-ratelimit-pb", - "//external:ares", - "//external:libssl", - "//external:nghttp2", - "//external:spdlog", - "//external:tclap", - "//external:lightstep", - "//external:event", - "//external:protobuf", - "//external:http_parser", - "//external:rapidjson", - "//external:event_pthreads", - ], -) - -cc_library( - name = "envoy-main", - srcs = [ - "source/exe/main.cc", - ], - copts = [ - "-I./external/envoy_git/source", - "-include ./external/envoy_git/source/precompiled/precompiled.h", - ], - deps = [ - ":envoy-common", - ], - linkstatic=1, -) - -cc_binary( - name = "envoy", - srcs = [ - "source/exe/main.cc", - ], - copts = [ - "-I./external/envoy_git/source", - "-include ./external/envoy_git/source/precompiled/precompiled.h", - ], - deps = [ - ":envoy-common", - ], - linkstatic=1, -) - -cc_library( - name = "envoy-test-lib", - srcs = glob([ - "test/**/*.cc", - "test/**/*.h", - ]), - copts = [ - "-I./external/envoy_git/source", - "-include ./external/envoy_git/test/precompiled/precompiled_test.h", - ], - includes = [ - "include", - ], - deps = [ - ":envoy-common", - ":envoy-test-pb", - "//external:googletest", - ], - alwayslink=1, -) - -filegroup( - name = "envoy-testdata", - srcs = glob([ - "generated/**/*", - "test/**/*", - ]), -) - -cc_test( - name = "envoy-test", - data = [ - ":envoy-testdata", - ], - deps = [ - ":envoy-test-lib", - ":envoy-test-pb", - "//external:googletest", - ], - linkstatic=1, -)""" - - native.new_git_repository( - name = "envoy_git", + # @boringssl is defined in //:repositories.bzl, but bound to libssl for + # grpc. Rebind to what envoy expects here. + native.bind( + name = "ssl", + actual = "@boringssl//:ssl", + ) + native.git_repository( + name = "envoy", remote = "https://github.com/lyft/envoy.git", - commit = "9dcac8ca111ecc8da059d1f8d42eb766b44bacd6", # https://github.com/lyft/envoy/pull/553 - build_file_content = BUILD, + commit = "bf3f23ad439ee83b91015dc4d0d7cb53b14bf1bc", ) diff --git a/src/envoy/transcoding/BUILD b/src/envoy/transcoding/BUILD new file mode 100644 index 00000000000..8e15a45eadb --- /dev/null +++ b/src/envoy/transcoding/BUILD @@ -0,0 +1,40 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +cc_library( + name = "envoy_input_stream", + srcs = [ + "envoy_input_stream.cc", + ], + hdrs = [ + "envoy_input_stream.h", + ], + deps = [ + "//contrib/endpoints/src/grpc/transcoding:transcoder_input_stream", + "@envoy//source/exe:envoy_common_lib", + ], +) + +cc_test( + name = "envoy_input_stream_test", + srcs = [ + "envoy_input_stream_test.cc", + ], + deps = [ + ":envoy_input_stream", + "@googletest_git//:googletest_main", + ], +) diff --git a/src/envoy/transcoding/envoy_input_stream.cc b/src/envoy/transcoding/envoy_input_stream.cc new file mode 100644 index 00000000000..81334303a7a --- /dev/null +++ b/src/envoy/transcoding/envoy_input_stream.cc @@ -0,0 +1,63 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/transcoding/envoy_input_stream.h" + +namespace Grpc { + +void EnvoyInputStream::Move(Buffer::Instance &instance) { + if (!finished_) { + buffer_.move(instance); + } +} + +bool EnvoyInputStream::Next(const void **data, int *size) { + if (position_ != 0) { + buffer_.drain(position_); + position_ = 0; + } + + Buffer::RawSlice slice; + uint64_t num_slices = buffer_.getRawSlices(&slice, 1); + + if (num_slices) { + *data = slice.mem_; + *size = slice.len_; + position_ = slice.len_; + byte_count_ += slice.len_; + return true; + } + + if (!finished_) { + *data = nullptr; + *size = 0; + return true; + } + return false; +} + +void EnvoyInputStream::BackUp(int count) { + GOOGLE_CHECK_GE(count, 0); + GOOGLE_CHECK_LE(count, position_); + + position_ -= count; + byte_count_ -= count; +} + +int64_t EnvoyInputStream::BytesAvailable() const { + return buffer_.length() - position_; +} + +} // namespace Grpc \ No newline at end of file diff --git a/src/envoy/transcoding/envoy_input_stream.h b/src/envoy/transcoding/envoy_input_stream.h new file mode 100644 index 00000000000..3e204e21eec --- /dev/null +++ b/src/envoy/transcoding/envoy_input_stream.h @@ -0,0 +1,51 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/buffer/buffer_impl.h" +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" + +namespace Grpc { + +class EnvoyInputStream + : public google::api_manager::transcoding::TranscoderInputStream { + public: + // Add a buffer to input stream, will consume all buffer from parameter + // if the stream is not finished + void Move(Buffer::Instance &instance); + + // Mark the buffer is finished + void Finish() { finished_ = true; } + + // TranscoderInputStream + virtual bool Next(const void **data, int *size) override; + virtual void BackUp(int count) override; + virtual bool Skip(int count) override { return false; } // Not implemented + virtual google::protobuf::int64 ByteCount() const override { + return byte_count_; + } + virtual int64_t BytesAvailable() const override; + + private: + Buffer::OwnedImpl buffer_; + int position_{0}; + int64_t byte_count_{0}; + bool finished_{false}; +}; + +} // namespace Grpc \ No newline at end of file diff --git a/src/envoy/transcoding/envoy_input_stream_test.cc b/src/envoy/transcoding/envoy_input_stream_test.cc new file mode 100644 index 00000000000..e8604367b91 --- /dev/null +++ b/src/envoy/transcoding/envoy_input_stream_test.cc @@ -0,0 +1,97 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/transcoding/envoy_input_stream.h" + +#include "gtest/gtest.h" + +namespace Grpc { +namespace { + +class EnvoyInputStreamTest : public testing::Test { + public: + EnvoyInputStreamTest() { + Buffer::OwnedImpl buffer{"abcd"}; + stream_.Move(buffer); + } + + std::string slice_data_{"abcd"}; + EnvoyInputStream stream_; + + const void *data_; + int size_; +}; + +TEST_F(EnvoyInputStreamTest, Move) { + Buffer::OwnedImpl buffer{"abcd"}; + stream_.Move(buffer); + + EXPECT_EQ(0, buffer.length()); + EXPECT_EQ(8, stream_.BytesAvailable()); +} + +TEST_F(EnvoyInputStreamTest, Next) { + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, size_); + EXPECT_EQ(0, memcmp(slice_data_.data(), data_, size_)); +} + +TEST_F(EnvoyInputStreamTest, TwoSlices) { + Buffer::OwnedImpl buffer("efgh"); + + stream_.Move(buffer); + + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, size_); + EXPECT_EQ(0, memcmp(slice_data_.data(), data_, size_)); + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, size_); + EXPECT_EQ(0, memcmp("efgh", data_, size_)); +} + +TEST_F(EnvoyInputStreamTest, BackUp) { + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, size_); + + stream_.BackUp(3); + EXPECT_EQ(3, stream_.BytesAvailable()); + EXPECT_EQ(1, stream_.ByteCount()); + + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(3, size_); + EXPECT_EQ(0, memcmp("bcd", data_, size_)); + EXPECT_EQ(4, stream_.ByteCount()); +} + +TEST_F(EnvoyInputStreamTest, ByteCount) { + EXPECT_EQ(0, stream_.ByteCount()); + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, stream_.ByteCount()); +} + +TEST_F(EnvoyInputStreamTest, Finish) { + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(0, size_); + stream_.Finish(); + EXPECT_FALSE(stream_.Next(&data_, &size_)); + + Buffer::OwnedImpl buffer("efgh"); + stream_.Move(buffer); + + EXPECT_EQ(4, buffer.length()); +} +} +} \ No newline at end of file diff --git a/.bazelrc b/tools/bazel.rc similarity index 100% rename from .bazelrc rename to tools/bazel.rc diff --git a/tools/bazel.rc.jenkins b/tools/bazel.rc.jenkins new file mode 100644 index 00000000000..664f7b13380 --- /dev/null +++ b/tools/bazel.rc.jenkins @@ -0,0 +1,8 @@ +# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. +startup --host_jvm_args=-Xmx8192m +startup --host_jvm_args=-Xms8192m +startup --batch + +# This is so we understand failures better +build --verbose_failures + diff --git a/.bazelrc.travis b/tools/bazel.rc.travis similarity index 100% rename from .bazelrc.travis rename to tools/bazel.rc.travis From b2b45fd38af4c9c54a62d6ea2a3a88bf18b1c809 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Wed, 12 Apr 2017 14:55:21 -0700 Subject: [PATCH 13/15] Allow HTTP functions in firebase rules to specify audience (#244) * Allow HTTP functions in firebase rules to specify audience * Allow GetAuthToken to ignore cache and fix style checks. * Fix GetAuthToken * Address Wayne's comment * Check for empty response body --- .../api_manager/auth/service_account_token.cc | 12 ++- .../api_manager/auth/service_account_token.h | 7 ++ .../src/api_manager/check_security_rules.cc | 20 +++-- .../api_manager/check_security_rules_test.cc | 76 +++++++++++-------- .../api_manager/context/service_context.cc | 15 ---- .../firebase_rules/firebase_request.cc | 19 ++++- .../firebase_rules/firebase_request.h | 1 + .../src/api_manager/proto/server_config.proto | 1 - 8 files changed, 95 insertions(+), 56 deletions(-) diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token.cc b/contrib/endpoints/src/api_manager/auth/service_account_token.cc index 69f5e8b126e..a0ff6d4cc87 100644 --- a/contrib/endpoints/src/api_manager/auth/service_account_token.cc +++ b/contrib/endpoints/src/api_manager/auth/service_account_token.cc @@ -56,10 +56,20 @@ Status ServiceAccountToken::SetClientAuthSecret(const std::string& secret) { void ServiceAccountToken::SetAudience(JWT_TOKEN_TYPE type, const std::string& audience) { GOOGLE_CHECK(type >= 0 && type < JWT_TOKEN_TYPE_MAX); - jwt_tokens_[type].set_audience(audience); + if (jwt_tokens_[type].audience() != audience) { + jwt_tokens_[type].set_token("", 0); + jwt_tokens_[type].set_audience(audience); + } } const std::string& ServiceAccountToken::GetAuthToken(JWT_TOKEN_TYPE type) { + return GetAuthToken(type, jwt_tokens_[type].audience()); +} + +const std::string& ServiceAccountToken::GetAuthToken( + JWT_TOKEN_TYPE type, const std::string& audience) { + SetAudience(type, audience); + // Uses authentication secret if available. if (!client_auth_secret_.empty()) { GOOGLE_CHECK(type >= 0 && type < JWT_TOKEN_TYPE_MAX); diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token.h b/contrib/endpoints/src/api_manager/auth/service_account_token.h index ef3cabce600..1946e4daaa6 100644 --- a/contrib/endpoints/src/api_manager/auth/service_account_token.h +++ b/contrib/endpoints/src/api_manager/auth/service_account_token.h @@ -79,6 +79,13 @@ class ServiceAccountToken { // Otherwise, use the access token fetched from metadata server. const std::string& GetAuthToken(JWT_TOKEN_TYPE type); + // Gets the auth token to access Google services. This method accepts an + // audience parameter to set when generating JWT token. + // If client auth secret is specified, use it to calcualte JWT token. + // Otherwise, use the access token fetched from metadata server. + const std::string& GetAuthToken(JWT_TOKEN_TYPE type, + const std::string& audience); + private: // Stores base token info. Used for both OAuth and JWT tokens. class TokenInfo { diff --git a/contrib/endpoints/src/api_manager/check_security_rules.cc b/contrib/endpoints/src/api_manager/check_security_rules.cc index 2bb5180e981..c5a2497e894 100644 --- a/contrib/endpoints/src/api_manager/check_security_rules.cc +++ b/contrib/endpoints/src/api_manager/check_security_rules.cc @@ -23,6 +23,9 @@ using ::google::api_manager::auth::GetStringValue; using ::google::api_manager::firebase_rules::FirebaseRequest; using ::google::api_manager::utils::Status; +const char kFirebaseAudience[] = + "https://staging-firebaserules.sandbox.googleapis.com/" + "google.firebase.rules.v1.FirebaseRulesService"; namespace google { namespace api_manager { @@ -77,6 +80,7 @@ class AuthzChecker : public std::enable_shared_from_this { void HttpFetch(const std::string &url, const std::string &method, const std::string &request_body, auth::ServiceAccountToken::JWT_TOKEN_TYPE token_type, + const std::string &audience, std::function continuation); std::shared_ptr GetPtr() { return shared_from_this(); } @@ -107,8 +111,8 @@ void AuthzChecker::Check( auto checker = GetPtr(); HttpFetch(GetReleaseUrl(*context), kHttpGetMethod, "", auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE, - [context, final_continuation, checker](Status status, - std::string &&body) { + kFirebaseAudience, [context, final_continuation, checker]( + Status status, std::string &&body) { std::string ruleset_id; if (status.ok()) { checker->env_->LogDebug( @@ -143,16 +147,17 @@ void AuthzChecker::CallNextRequest( auto checker = GetPtr(); firebase_rules::HttpRequest http_request = request_handler_->GetHttpRequest(); HttpFetch(http_request.url, http_request.method, http_request.body, - http_request.token_type, + http_request.token_type, http_request.audience, [continuation, checker](Status status, std::string &&body) { checker->env_->LogError(std::string("Response Body = ") + body); - if (status.ok()) { + if (status.ok() && !body.empty()) { checker->request_handler_->UpdateResponse(body); checker->CallNextRequest(continuation); } else { - checker->env_->LogError(std::string("Test API failed with ") + - status.ToString()); + checker->env_->LogError( + std::string("Test API failed with ") + + (status.ok() ? "Empty Response" : status.ToString())); status = Status(Code::INTERNAL, kFailedFirebaseTest); continuation(status); } @@ -187,6 +192,7 @@ void AuthzChecker::HttpFetch( const std::string &url, const std::string &method, const std::string &request_body, auth::ServiceAccountToken::JWT_TOKEN_TYPE token_type, + const std::string &audience, std::function continuation) { env_->LogDebug(std::string("Issue HTTP Request to url :") + url + " method : " + method + " body: " + request_body); @@ -201,7 +207,7 @@ void AuthzChecker::HttpFetch( } request->set_method(method).set_url(url).set_auth_token( - sa_token_->GetAuthToken(token_type)); + sa_token_->GetAuthToken(token_type, audience)); if (!request_body.empty()) { request->set_header(kContentType, kApplication).set_body(request_body); diff --git a/contrib/endpoints/src/api_manager/check_security_rules_test.cc b/contrib/endpoints/src/api_manager/check_security_rules_test.cc index bf60d21bd60..cefab67fbab 100644 --- a/contrib/endpoints/src/api_manager/check_security_rules_test.cc +++ b/contrib/endpoints/src/api_manager/check_security_rules_test.cc @@ -43,7 +43,7 @@ using ::google::protobuf::RepeatedPtrField; // Tuple with arg<0> = function name // arg<1> = url, arg<2> = method, arg<3> = body. using FuncTuple = - std::tuple; + std::tuple; using ::google::api_manager::proto::TestRulesetResponse; using FunctionCall = TestRulesetResponse::TestResult::FunctionCall; @@ -157,14 +157,16 @@ static const char kDummyBody[] = R"( "key" : "value" })"; +static const char kDummyAudience[] = "test-audience"; + const char kFirstRequest[] = R"({"testSuite":{"testCases":[{"expectation":"ALLOW","request":{"method":"get","path":"/ListShelves","auth":{"token":{"email":"limin-429@appspot.gserviceaccount.com","email_verified":true,"azp":"limin-429@appspot.gserviceaccount.com","aud":"https://myfirebaseapp.appspot.com","sub":"113424383671131376652","iat":1486575396,"iss":"https://accounts.google.com","exp":1486578996}}}}]}})"; const char kSecondRequest[] = - R"({"testSuite":{"testCases":[{"expectation":"ALLOW","request":{"auth":{"token":{"email":"limin-429@appspot.gserviceaccount.com","azp":"limin-429@appspot.gserviceaccount.com","aud":"https://myfirebaseapp.appspot.com","sub":"113424383671131376652","iss":"https://accounts.google.com","email_verified":true,"iat":1486575396,"exp":1486578996}},"method":"get","path":"/ListShelves"},"functionMocks":[{"function":"f1","args":[{"exactValue":"http://url1"},{"exactValue":"POST"},{"exactValue":{"key":"value"}}],"result":{"value":{"key":"value"}}}]}]}})"; + R"({"testSuite":{"testCases":[{"expectation":"ALLOW","request":{"auth":{"token":{"email":"limin-429@appspot.gserviceaccount.com","azp":"limin-429@appspot.gserviceaccount.com","aud":"https://myfirebaseapp.appspot.com","sub":"113424383671131376652","iss":"https://accounts.google.com","email_verified":true,"iat":1486575396,"exp":1486578996}},"method":"get","path":"/ListShelves"},"functionMocks":[{"function":"f1","args":[{"exactValue":"http://url1"},{"exactValue":"POST"},{"exactValue":{"key":"value"}},{"exactValue":"test-audience"}],"result":{"value":{"key":"value"}}}]}]}})"; const char kThirdRequest[] = - R"({"testSuite":{"testCases":[{"expectation":"ALLOW","request":{"method":"get","path":"/ListShelves","auth":{"token":{"email":"limin-429@appspot.gserviceaccount.com","iat":1486575396,"azp":"limin-429@appspot.gserviceaccount.com","exp":1486578996,"email_verified":true,"sub":"113424383671131376652","aud":"https://myfirebaseapp.appspot.com","iss":"https://accounts.google.com"}}},"functionMocks":[{"function":"f2","args":[{"exactValue":"http://url2"},{"exactValue":"GET"},{"exactValue":{"key":"value"}}],"result":{"value":{"key":"value"}}},{"function":"f3","args":[{"exactValue":"https://url3"},{"exactValue":"GET"},{"exactValue":{"key":"value"}}],"result":{"value":{"key":"value"}}},{"function":"f1","args":[{"exactValue":"http://url1"},{"exactValue":"POST"},{"exactValue":{"key":"value"}}],"result":{"value":{"key":"value"}}}]}]}})"; + R"({"testSuite":{"testCases":[{"expectation":"ALLOW","request":{"method":"get","path":"/ListShelves","auth":{"token":{"email":"limin-429@appspot.gserviceaccount.com","iat":1486575396,"azp":"limin-429@appspot.gserviceaccount.com","exp":1486578996,"email_verified":true,"sub":"113424383671131376652","aud":"https://myfirebaseapp.appspot.com","iss":"https://accounts.google.com"}}},"functionMocks":[{"function":"f2","args":[{"exactValue":"http://url2"},{"exactValue":"GET"},{"exactValue":{"key":"value"}},{"exactValue":"test-audience"}],"result":{"value":{"key":"value"}}},{"function":"f3","args":[{"exactValue":"https://url3"},{"exactValue":"GET"},{"exactValue":{"key":"value"}},{"exactValue":"test-audience"}],"result":{"value":{"key":"value"}}},{"function":"f1","args":[{"exactValue":"http://url1"},{"exactValue":"POST"},{"exactValue":{"key":"value"}},{"exactValue":"test-audience"}],"result":{"value":{"key":"value"}}}]}]}})"; ::google::protobuf::Value ToValue(const std::string &arg) { ::google::protobuf::Value value; @@ -197,7 +199,8 @@ MATCHER_P3(HTTPRequestMatches, url, method, body, "") { } FunctionCall BuildCall(const std::string &name, const std::string &url, - const std::string &method, const std::string &body) { + const std::string &method, const std::string &body, + const std::string &audience) { FunctionCall func_call; func_call.set_function(name); @@ -213,6 +216,10 @@ FunctionCall BuildCall(const std::string &name, const std::string &url, *(func_call.add_args()) = ToValue(body); } + if (!audience.empty()) { + *(func_call.add_args()) = ToValue(audience); + } + return func_call; } @@ -386,6 +393,10 @@ class CheckSecurityRulesTest : public ::testing::Test { Status status = utils::JsonToProto(std::get<3>(http), &body); *(func->add_args()) = body; } + + if (!std::get<4>(http).empty()) { + func->add_args()->set_string_value(std::get<4>(http)); + } } std::string json_str; @@ -526,26 +537,30 @@ class CheckSecurityRulesFunctions : public CheckSecurityRulesTest, InSequence s; ExpectCall(release_url_, "GET", "", kRelease); - ExpectCall( - ruleset_test_url_, "POST", kFirstRequest, - BuildTestRulesetResponse( - false, {std::make_tuple("f1", "http://url1", "POST", kDummyBody)})); + ExpectCall(ruleset_test_url_, "POST", kFirstRequest, + BuildTestRulesetResponse( + false, {std::make_tuple("f1", "http://url1", "POST", + kDummyBody, kDummyAudience)})); ExpectCall("http://url1", "POST", kDummyBody, kDummyBody); - ExpectCall( - ruleset_test_url_, "POST", kSecondRequest, - BuildTestRulesetResponse( - false, {std::make_tuple("f2", "http://url2", "GET", kDummyBody), - std::make_tuple("f3", "https://url3", "GET", kDummyBody), - std::make_tuple("f1", "http://url1", "POST", kDummyBody)})); + ExpectCall(ruleset_test_url_, "POST", kSecondRequest, + BuildTestRulesetResponse( + false, {std::make_tuple("f2", "http://url2", "GET", + kDummyBody, kDummyAudience), + std::make_tuple("f3", "https://url3", "GET", + kDummyBody, kDummyAudience), + std::make_tuple("f1", "http://url1", "POST", + kDummyBody, kDummyAudience)})); ExpectCall("http://url2", "GET", kDummyBody, kDummyBody); ExpectCall("https://url3", "GET", kDummyBody, kDummyBody); ExpectCall(ruleset_test_url_, "POST", kThirdRequest, BuildTestRulesetResponse( - GetParam(), - {std::make_tuple("f2", "http://url2", "GET", kDummyBody), - std::make_tuple("f3", "https://url3", "GET", kDummyBody), - std::make_tuple("f1", "http://url1", "POST", kDummyBody)})); + GetParam(), {std::make_tuple("f2", "http://url2", "GET", + kDummyBody, kDummyAudience), + std::make_tuple("f3", "https://url3", "GET", + kDummyBody, kDummyAudience), + std::make_tuple("f1", "http://url1", "POST", + kDummyBody, kDummyAudience)})); } }; @@ -611,18 +626,19 @@ TEST_P(CheckSecurityRulesBadFunctions, CheckBadFunctionArguments) { }); } -INSTANTIATE_TEST_CASE_P(CheckSecurityRulesBadFunctionArguments, - CheckSecurityRulesBadFunctions, - ::testing::Values( - // Empty function name - std::make_tuple("", "http://url1", "POST", - kDummyBody), - // Argument count less than 2 - std::make_tuple("f1", "", "", ""), - // The url is not set - std::make_tuple("f1", "", "POST", kDummyBody), - // The url is not a http or https protocol - std::make_tuple("f1", "ftp://url1", "BODY", ""))); +INSTANTIATE_TEST_CASE_P( + CheckSecurityRulesBadFunctionArguments, CheckSecurityRulesBadFunctions, + ::testing::Values( + // Empty function name + std::make_tuple("", "http://url1", "POST", kDummyBody, kDummyAudience), + // Argument count less than 3 + std::make_tuple("f1", "http://url1", "", "", kDummyAudience), + // The url is not set + std::make_tuple("f1", "", "POST", kDummyBody, kDummyAudience), + // The url is not a http or https protocol + std::make_tuple("f1", "ftp://url1", "POST", kDummyBody, kDummyAudience), + // The audience is not present + std::make_tuple("f1", "http://url1", "GET", kDummyBody, ""))); } } // namespace api_manager } // namespace google diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index 8e1493d3cfa..92b850481e7 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -73,21 +73,6 @@ ServiceContext::ServiceContext(std::unique_ptr env, ->service_control_config() .intermediate_report_min_interval(); } - - service_account_token_.SetAudience( - auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE, kFirebaseAudience); - - if (config_->server_config() && - !config_->server_config() - ->api_check_security_rules_config() - .authorization_service_audience() - .empty()) { - service_account_token_.SetAudience( - auth::ServiceAccountToken::JWT_TOKEN_FOR_AUTHORIZATION_SERVICE, - config_->server_config() - ->api_check_security_rules_config() - .authorization_service_audience()); - } } MethodCallInfo ServiceContext::GetMethodCallInfo( diff --git a/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.cc b/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.cc index 300d521ed1f..7a69cbed47c 100644 --- a/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.cc +++ b/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.cc @@ -49,6 +49,9 @@ const std::string kFirebaseDeleteMethod = "delete"; const std::string kFirebaseUpdateMethod = "update"; const std::string kV1 = "/v1"; const std::string kTestQuery = ":test?alt=json"; +const char kFirebaseAudience[] = + "https://staging-firebaserules.sandbox.googleapis.com/" + "google.firebase.rules.v1.FirebaseRulesService"; void SetProtoValue(const std::string &key, const ::google::protobuf::Value &value, @@ -94,6 +97,8 @@ FirebaseRequest::FirebaseRequest( firebase_http_request_.method = kHttpPostMethod; firebase_http_request_.token_type = auth::ServiceAccountToken::JWT_TOKEN_FOR_FIREBASE; + firebase_http_request_.audience = kFirebaseAudience; + external_http_request_.token_type = auth::ServiceAccountToken::JWT_TOKEN_FOR_AUTHORIZATION_SERVICE; @@ -305,6 +310,8 @@ Status FirebaseRequest::SetNextRequest() { auto call = *func_call_iter_; external_http_request_.url = call.args(0).string_value(); external_http_request_.method = call.args(1).string_value(); + external_http_request_.audience = + call.args(call.args_size() - 1).string_value(); std::string body; status = utils::ProtoToJson(call.args(2), &body, utils::JsonOptions::DEFAULT); @@ -334,9 +341,9 @@ Status FirebaseRequest::CheckFuncCallArgs(const FunctionCall &func) { // We only support functions that call with three argument: HTTP URL, HTTP // method and body. The body can be empty - if (func.args_size() < 2 || func.args_size() > 3) { + if (func.args_size() < 3 || func.args_size() > 4) { std::ostringstream os; - os << func.function() << " Require 2 or 3 arguments. But has " + os << func.function() << " Require 3 or 4 arguments. But has " << func.args_size(); return Status(Code::INVALID_ARGUMENT, os.str()); } @@ -348,6 +355,14 @@ Status FirebaseRequest::CheckFuncCallArgs(const FunctionCall &func) { std::string(func.function() + " Arguments 1 and 2 should be strings")); } + if (func.args(func.args_size() - 1).kind_case() != + google::protobuf::Value::kStringValue) { + return Status( + Code::INVALID_ARGUMENT, + std::string(func.function() + "The last argument should be a string" + "that specifies audience")); + } + if (!utils::IsHttpRequest(func.args(0).string_value())) { return Status( Code::INVALID_ARGUMENT, diff --git a/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h b/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h index ee26b77e646..1a9a22ea699 100644 --- a/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h +++ b/contrib/endpoints/src/api_manager/firebase_rules/firebase_request.h @@ -74,6 +74,7 @@ struct HttpRequest { std::string method; std::string body; auth::ServiceAccountToken::JWT_TOKEN_TYPE token_type; + std::string audience; }; // A FirebaseRequest object understands the various http requests that need diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index 5e26613f140..1ec8eefe5d5 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -164,7 +164,6 @@ message ApiAuthenticationConfig { message ApiCheckSecurityRulesConfig { // Firebase server to use. string firebase_server = 1; - string authorization_service_audience = 2; } message Experimental { From 78fee54941fbcbb7d925b6199b4ce92285a1bb62 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Mon, 24 Apr 2017 10:01:33 -0700 Subject: [PATCH 14/15] Remove .bazelrc.jenkins file not present in the master branch. --- .bazelrc.jenkins | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .bazelrc.jenkins diff --git a/.bazelrc.jenkins b/.bazelrc.jenkins deleted file mode 100644 index 664f7b13380..00000000000 --- a/.bazelrc.jenkins +++ /dev/null @@ -1,8 +0,0 @@ -# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. -startup --host_jvm_args=-Xmx8192m -startup --host_jvm_args=-Xms8192m -startup --batch - -# This is so we understand failures better -build --verbose_failures - From 990d8e09e33ac7cd53a69805ce9283c9e5746af0 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Mon, 24 Apr 2017 10:05:26 -0700 Subject: [PATCH 15/15] Remove forward_attribute_filter.cc not present in master. --- src/envoy/mixer/forward_attribute_filter.cc | 114 -------------------- 1 file changed, 114 deletions(-) delete mode 100644 src/envoy/mixer/forward_attribute_filter.cc diff --git a/src/envoy/mixer/forward_attribute_filter.cc b/src/envoy/mixer/forward_attribute_filter.cc deleted file mode 100644 index 08d58916f89..00000000000 --- a/src/envoy/mixer/forward_attribute_filter.cc +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright 2017 Istio Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "precompiled/precompiled.h" - -#include "common/common/base64.h" -#include "common/common/logger.h" -#include "common/http/headers.h" -#include "common/http/utility.h" -#include "envoy/server/instance.h" -#include "server/config/network/http_connection_manager.h" -#include "src/envoy/mixer/utils.h" - -namespace Http { -namespace ForwardAttribute { -namespace { - -// The Json object name to specify attributes which will be forwarded -// to the upstream istio proxy. -const std::string kJsonNameAttributes("attributes"); - -} // namespace - -class Config : public Logger::Loggable { - private: - std::string attributes_; - - public: - Config(const Json::Object& config) { - Utils::StringMap attributes = - Utils::ExtractStringMap(config, kJsonNameAttributes); - if (!attributes.empty()) { - std::string serialized_str = Utils::SerializeStringMap(attributes); - attributes_ = - Base64::encode(serialized_str.c_str(), serialized_str.size()); - } - } - - const std::string& attributes() const { return attributes_; } -}; - -typedef std::shared_ptr ConfigPtr; - -class ForwardAttributeFilter : public Http::StreamDecoderFilter { - private: - ConfigPtr config_; - - public: - ForwardAttributeFilter(ConfigPtr config) : config_(config) {} - - FilterHeadersStatus decodeHeaders(HeaderMap& headers, - bool end_stream) override { - if (!config_->attributes().empty()) { - headers.addStatic(Utils::kIstioAttributeHeader, config_->attributes()); - } - return FilterHeadersStatus::Continue; - } - - FilterDataStatus decodeData(Buffer::Instance& data, - bool end_stream) override { - return FilterDataStatus::Continue; - } - - FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - return FilterTrailersStatus::Continue; - } - - void setDecoderFilterCallbacks( - StreamDecoderFilterCallbacks& callbacks) override {} -}; - -} // namespace ForwardAttribute -} // namespace Http - -namespace Server { -namespace Configuration { - -class ForwardAttributeConfig : public HttpFilterConfigFactory { - public: - HttpFilterFactoryCb tryCreateFilterFactory( - HttpFilterType type, const std::string& name, const Json::Object& config, - const std::string&, Server::Instance& server) override { - if (type != HttpFilterType::Decoder || name != "forward_attribute") { - return nullptr; - } - - Http::ForwardAttribute::ConfigPtr add_header_config( - new Http::ForwardAttribute::Config(config)); - return [add_header_config]( - Http::FilterChainFactoryCallbacks& callbacks) -> void { - std::shared_ptr instance( - new Http::ForwardAttribute::ForwardAttributeFilter( - add_header_config)); - callbacks.addStreamDecoderFilter(Http::StreamDecoderFilterPtr(instance)); - }; - } -}; - -static RegisterHttpFilterConfigFactory register_; - -} // namespace Configuration -} // namespace server