Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sqlproxyccl: add proxy protocol listener #117241

Merged
merged 1 commit into from
Jan 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/ccl/cliccl/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func init() {
cliflagcfg.StringFlag(f, &proxyContext.Denylist, cliflags.DenyList)
cliflagcfg.StringFlag(f, &proxyContext.Allowlist, cliflags.AllowList)
cliflagcfg.StringFlag(f, &proxyContext.ListenAddr, cliflags.ProxyListenAddr)
cliflagcfg.StringFlag(f, &proxyContext.ProxyProtocolListenAddr, cliflags.ProxyProtocolListenAddr)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The combination of flags here is a bit redundant. What do you think of making the existing --listen-addr flag as optional and adding a TODO to delete the require proxy protocol flag once we migrate everything over to --proxy-protocol?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

cliflagcfg.StringFlag(f, &proxyContext.ListenCert, cliflags.ListenCert)
cliflagcfg.StringFlag(f, &proxyContext.ListenKey, cliflags.ListenKey)
cliflagcfg.StringFlag(f, &proxyContext.MetricsAddress, cliflags.ListenMetrics)
Expand Down
30 changes: 23 additions & 7 deletions pkg/ccl/cliccl/mt_proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,20 @@ func runStartSQLProxy(cmd *cobra.Command, args []string) (returnErr error) {

log.Infof(ctx, "New proxy with opts: %+v", proxyContext)

proxyLn, err := net.Listen("tcp", proxyContext.ListenAddr)
if err != nil {
return err
var proxyLn net.Listener
if proxyContext.ListenAddr != "" {
proxyLn, err = net.Listen("tcp", proxyContext.ListenAddr)
if err != nil {
return err
}
}

var proxyProtocolLn net.Listener
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 to Jay's request for tests. That may be easier if you move most of this logic into the pkg/ccl/sqlproxyccl package that contains the majority of the sql proxy code.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've moved some logic around and added a test for how the main listener and the proxy protocol listener interact with the presence or absence of proxy headers. PTAL!

if proxyContext.ProxyProtocolListenAddr != "" {
proxyProtocolLn, err = net.Listen("tcp", proxyContext.ProxyProtocolListenAddr)
if err != nil {
return err
}
}

metricsLn, err := net.Listen("tcp", proxyContext.MetricsAddress)
Expand All @@ -84,15 +95,14 @@ func runStartSQLProxy(cmd *cobra.Command, args []string) (returnErr error) {
}

if err := stopper.RunAsyncTask(ctx, "serve-proxy", func(ctx context.Context) {
log.Infof(ctx, "proxy server listening at %s", proxyLn.Addr())
if err := server.Serve(ctx, proxyLn); err != nil {
if err := server.ServeSQL(ctx, proxyLn, proxyProtocolLn); err != nil {
errChan <- err
}
}); err != nil {
return err
}

return waitForSignals(ctx, server, stopper, proxyLn, errChan)
return waitForSignals(ctx, server, stopper, proxyLn, proxyProtocolLn, errChan)
}

func initLogging(cmd *cobra.Command) (ctx context.Context, stopper *stop.Stopper, err error) {
Expand All @@ -110,6 +120,7 @@ func waitForSignals(
server *sqlproxyccl.Server,
stopper *stop.Stopper,
proxyLn net.Listener,
proxyProtocolLn net.Listener,
errChan chan error,
) (returnErr error) {
// Need to alias the signals if this has to run on non-unix OSes too.
Expand Down Expand Up @@ -139,7 +150,12 @@ func waitForSignals(
// waiting for "shutdownConnectionTimeout" to elapse after which
// open TCP connections will be forcefully closed so the server can stop
log.Infof(ctx, "stopping tcp listener")
_ = proxyLn.Close()
if proxyLn != nil {
_ = proxyLn.Close()
}
if proxyProtocolLn != nil {
_ = proxyProtocolLn.Close()
}
select {
case <-server.AwaitNoConnections(ctx):
case <-time.After(shutdownConnectionTimeout):
Expand Down
10 changes: 8 additions & 2 deletions pkg/ccl/sqlproxyccl/proxy_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ type ProxyOptions struct {
Denylist string
// ListenAddr is the listen address for incoming connections.
ListenAddr string
// ProxyProtocolListenAddr is the optional listen address for incoming
// connections for which it will be enforced that the connections have proxy
// headers set.
ProxyProtocolListenAddr string
// ListenCert is the file containing PEM-encoded x509 certificate for listen
// address. Set to "*" to auto-generate self-signed cert.
ListenCert string
Expand Down Expand Up @@ -113,8 +117,10 @@ type ProxyOptions struct {
DisableConnectionRebalancing bool
// RequireProxyProtocol changes the server's behavior to support the PROXY
// protocol (SQL=required, HTTP=best-effort). With this set to true, the
// PROXY info from upstream will be trusted on both HTTP and SQL, if the
// headers are allowed.
// PROXY info from upstream will be trusted on both HTTP and SQL (on the
// ListenAddr port), if the headers are allowed. The ProxyProtocolListenAddr
// port, if specified, will require the proxy protocol regardless of
// RequireProxyProtocol.
RequireProxyProtocol bool

// testingKnobs are knobs used for testing.
Expand Down
Loading