refactor KES API and internals

**Description:**

This commit introduces a series of significant changes to various
components within the KES project. Among other things it:
1. Exposes a top-level library API for running and customizing
   KES servers.
2. Improves logging by using structured logging (`log/slog`).
3. Removes unused code
4. Introduces a KES-specific framework for handling HTTP requests
   (`internal/api`).
5. Stabilizes the KES API and prepares the introduction of
   protobuf as serialization format (in addition to JSON).

However, this commit does not refactor the `kv` package or
the KES config file handling. While still required, this will
be done in a separate commit.

**Performance**

A lot of effort has gone into designing and implementing an efficient
KES library API. Since majority of KES operations are read-only,
accessing a policy, encrypting a message, a.s.o., it can leverage and
benefit from lock-free concurrency primitives. Hence, the `Server`
type tries to avoid blocking on `sync.{RW}Mutex` as much as possible
and instead uses atomic primitives, like `atomic.Pointer`.

Further, the logging framework has been completely reworked to use
structured logging using the `log/slog` standard library package.
Now, error log messages are only generated when required (based on
log levels). The audit logging framework (`AuditHandler` and
`AuditRecord` type) works similar to the `slog` package and is also
designed to be efficient.

**Readability**

The new `internal/api` package provides a small KES-specific framework
for defining HTTP APIs and handling request. It tries to provide
composable primitives to build HTTP APIs that are efficient, secure
and easy to reason about. It provides a specific `Request` type that
represents an authenticated HTTP request. This allows to separate
buisness logic (e.g. handling a key creation request) from timeout
handling, authentication, etc.

Further, this commit tries to add more expressive documentation
describing the intent.

**Versioning**

The KES library package will follow semantic versioning, like any
other Go module. However, the KES server command and CLI (`cmd/kes`)
will continue to use the rolling release timestamp versioning.
A KES library release can be tagged independently from the KES
CLI and vice versa. Users of the KES package will be able to
import like any other Go module: `import "github.com/minio/[email protected]"`.

Signed-off-by: Andreas Auernhammer <[email protected]>

audit.go

@@ -0,0 +1,188 @@
+// Copyright 2023 - MinIO, Inc. All rights reserved.
+// Use of this source code is governed by the AGPLv3
+// license that can be found in the LICENSE file.
+
+package kes
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/netip"
+	"time"
+
+	"github.com/minio/kes-go"
+	"github.com/minio/kes/internal/api"
+)
+
+// AuditRecord describes an audit event logged by a KES server.
+type AuditRecord struct {
+	// Point in time when the audit event happened.
+	Time time.Time
+
+	// The request HTTP method. (GET, PUT, ...)
+	Method string
+
+	// Request URL path. Always starts with a '/'.
+	Path string
+
+	// Identity that send the request.
+	Identity kes.Identity
+
+	// IP address of the client that sent the request.
+	RemoteIP netip.Addr
+
+	// Status code the KES server responded with.
+	StatusCode int
+
+	// Amount of time the server took to process the
+	// request and generate a response.
+	ResponseTime time.Duration
+
+	// The log level of this event.
+	Level slog.Level
+
+	// The log message describing the event.
+	Message string
+}
+
+// An AuditHandler handles audit records produced by a Server.
+//
+// A typical handler may print audit records to standard error,
+// or write them to a file or database.
+//
+// Any of the AuditHandler's methods may be called concurrently
+// with itself or with other methods. It is the responsibility
+// of the Handler to manage this concurrency.
+type AuditHandler interface {
+	// Enabled reports whether the handler handles records at
+	// the given level. The handler ignores records whose level
+	// is lower. It is called early, before an audit record is
+	// created, to safe effort if the audit event should be
+	// discarded.
+	//
+	// The Server will pass the request context as the first
+	// argument, or context.Background() if no context is
+	// available. Enabled may use the context to make a
+	// decision.
+	Enabled(context.Context, slog.Level) bool
+
+	// Handle handles the AuditRecord. It will only called when
+	// Enabled returns true.
+	//
+	// The context is present for providing AuditHandlers access
+	// to the context's values and to potentially pass it to an
+	// underlying slog.Handler. Canceling the context should not
+	// affect record processing.
+	Handle(context.Context, AuditRecord) error
+}
+
+// AuditLogHandler is an AuditHandler adapter that wraps
+// an slog.Handler. It converts AuditRecords to slog.Records
+// and passes them to the slog.Handler. An AuditLogHandler
+// acts as a bridge between AuditHandlers and slog.Handlers.
+type AuditLogHandler struct {
+	Handler slog.Handler
+}
+
+// Enabled reports whether the AuditLogHandler handles records
+// at the given level. It returns true if the underlying handler
+// returns true.
+func (a *AuditLogHandler) Enabled(ctx context.Context, level slog.Level) bool {
+	return a.Handler.Enabled(ctx, level)
+}
+
+// Handle converts the AuditRecord to an slog.Record and
+// passes it to the underlying handler.
+func (a *AuditLogHandler) Handle(ctx context.Context, r AuditRecord) error {
+	rec := slog.Record{
+		Time:    r.Time,
+		Message: r.Message,
+		Level:   r.Level,
+	}
+	rec.AddAttrs(
+		slog.Attr{Key: "req", Value: slog.GroupValue(
+			slog.String("method", r.Method),
+			slog.String("path", r.Path),
+			slog.String("ip", r.RemoteIP.String()),
+			slog.String("identity", r.Identity.String()),
+		)},
+		slog.Attr{Key: "res", Value: slog.GroupValue(
+			slog.Int("code", r.StatusCode),
+			slog.Duration("time", r.ResponseTime),
+		)},
+	)
+	return a.Handler.Handle(ctx, rec)
+}
+
+// An auditLogger records information about a request/response
+// handled by the Server.
+//
+// For each call of its Log method, it creates an AuditRecord and
+// passes it to its AuditHandler. If clients have subscribed to
+// the AuditLog API, the logger also sends the AuditRecord to these
+// clients.
+type auditLogger struct {
+	h     AuditHandler
+	level slog.Leveler
+
+	out *api.Multicast // clients subscribed to the AuditLog API
+}
+
+// newAuditLogger returns a new auditLogger passing AuditRecords to h.
+// A record is only sent to clients subscribed to the AuditLog API if
+// its log level is >= level.
+func newAuditLogger(h AuditHandler, level slog.Leveler) *auditLogger {
+	return &auditLogger{
+		h:     h,
+		level: level,
+		out:   &api.Multicast{},
+	}
+}
+
+// Log emits an audit record with the current time, log message,
+// response status code and request information.
+func (a *auditLogger) Log(msg string, statusCode int, req *api.Request) {
+	const Level = slog.LevelInfo
+	if Level < a.level.Level() {
+		return
+	}
+
+	hEnabled, oEnabled := a.h.Enabled(req.Context(), Level), a.out.Num() > 0
+	if !hEnabled && !oEnabled {
+		return
+	}
+
+	now := time.Now()
+	remoteIP, _ := netip.ParseAddrPort(req.RemoteAddr)
+	r := AuditRecord{
+		Time:         time.Now(),
+		Method:       req.Method,
+		Path:         req.URL.Path,
+		Identity:     req.Identity,
+		RemoteIP:     remoteIP.Addr(),
+		StatusCode:   statusCode,
+		ResponseTime: now.Sub(req.Received),
+		Level:        Level,
+		Message:      msg,
+	}
+	if hEnabled {
+		a.h.Handle(req.Context(), r)
+	}
+
+	if !oEnabled {
+		return
+	}
+	json.NewEncoder(a.out).Encode(api.AuditLogEvent{
+		Time: r.Time,
+		Request: api.AuditLogRequest{
+			IP:       r.RemoteIP.String(),
+			APIPath:  r.Path,
+			Identity: r.Identity.String(),
+		},
+		Response: api.AuditLogResponse{
+			StatusCode: r.StatusCode,
+			Time:       r.ResponseTime.Milliseconds(),
+		},
+	})
+}

auth.go

@@ -0,0 +1,173 @@
+// Copyright 2023 - MinIO, Inc. All rights reserved.
+// Use of this source code is governed by the AGPLv3
+// license that can be found in the LICENSE file.
+
+package kes
+
+import (
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"sync/atomic"
+
+	"github.com/minio/kes-go"
+	"github.com/minio/kes/internal/api"
+)
+
+// verifyIdentity authenticates client requests by verifying that
+// the client provides a certificate during the TLS handshake (mTLS)
+// and that the identity of the certificate public key matches either
+// the admin identity or an identity with an assigned policy.
+//
+// A request is accepted if the identity matches the admin identity
+// or the policy associated to the identity allows the request. The
+// later is the case if none of the policy's deny rules and at least
+// one of the policy's allow rules apply. Otherwise, the request is
+// rejected.
+type verifyIdentity atomic.Pointer[serverState]
+
+// Authenticate verifies that the request is either sent by the
+// server admin or passes the policy assigned to the identity.
+// Otherwise, it returns an error.
+func (v *verifyIdentity) Authenticate(req *http.Request) (*api.Request, api.Error) {
+	identity, err := identifyRequest(req.TLS)
+	if err != nil {
+		s := (*atomic.Pointer[serverState])(v).Load()
+		s.Log.DebugContext(req.Context(), err.Error(), "req", req)
+		return nil, err
+	}
+
+	s := (*atomic.Pointer[serverState])(v).Load()
+	if identity == s.Admin {
+		return &api.Request{
+			Request:  req,
+			Identity: identity,
+		}, nil
+	}
+
+	policy, ok := s.Identities[identity]
+	if !ok {
+		s.Log.DebugContext(req.Context(), "access denied: identity not found", "req", req)
+		return nil, kes.ErrNotAllowed
+	}
+	if err := policy.Verify(req); err != nil {
+		s.Log.DebugContext(req.Context(), fmt.Sprintf("access denied: rejected by policy '%s'", policy.Name), "req", req)
+		return nil, kes.ErrNotAllowed
+	}
+
+	return &api.Request{
+		Request:  req,
+		Identity: identity,
+	}, nil
+}
+
+// insecureIdentifyOnly does not authenticate client requests but
+// computes the certificate public key identity, if provided.
+// It does not return an error if the client did not provide a
+// certificate, or an invalid one, during the TLS handshake. In
+// such a case, the identity of the returned request is empty.
+type insecureIdentifyOnly struct{}
+
+func (insecureIdentifyOnly) Authenticate(req *http.Request) (*api.Request, api.Error) {
+	identity, _ := identifyRequest(req.TLS)
+	return &api.Request{
+		Request:  req,
+		Identity: identity,
+	}, nil
+}
+
+func identifyRequest(state *tls.ConnectionState) (kes.Identity, api.Error) {
+	if state == nil {
+		return "", api.NewError(http.StatusBadRequest, "insecure connection: TLS is required")
+	}
+
+	var cert *x509.Certificate
+	for _, c := range state.PeerCertificates {
+		if c.IsCA {
+			continue
+		}
+		if cert != nil {
+			return "", api.NewError(http.StatusBadRequest, "tls: received more than one client certificate")
+		}
+		cert = c
+	}
+	if cert == nil {
+		return "", api.NewError(http.StatusBadRequest, "tls: client certificate is required")
+	}
+
+	h := sha256.Sum256(cert.RawSubjectPublicKeyInfo)
+	return kes.Identity(hex.EncodeToString(h[:])), nil
+}
+
+// validName reports whether s is a valid {policy|identity|key} name.
+//
+// Valid names only contain the characters:
+//   - 0-9
+//   - A-Z
+//   - a-z
+//   - '-'  (hyphen, must not be first/last character)
+//   - '_'  (underscore, must not be the only character)
+//
+// More characters may be allowed in the future.
+func validName(s string) bool {
+	const MaxLength = 80 // Some arbitrary but reasonable limit
+
+	if s == "" || s == "_" || len(s) > MaxLength {
+		return false
+	}
+
+	n := len(s) - 1
+	for i, r := range s {
+		switch {
+		case r >= '0' && r <= '9':
+		case r >= 'A' && r <= 'Z':
+		case r >= 'a' && r <= 'z':
+		case r == '-' && i > 0 && i < n:
+		case r == '_':
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+// validPattern reports whether s is a valid pattern for
+// listing {policy|identity|key} names.
+//
+// Valid patterns only contain the characters:
+//   - 0-9
+//   - A-Z
+//   - a-z
+//   - '-'  (hyphen, must not be first/last character)
+//   - '_'  (underscore, must not be the only character)
+//   - '*'  (only as last character)
+//
+// More characters may be allowed in the future.
+func validPattern(s string) bool {
+	const MaxLength = 80 // Some arbitrary but reasonable limit
+
+	if s == "*" { // fast path
+		return true
+	}
+	if s == "_" || len(s) > MaxLength {
+		return false
+	}
+
+	n := len(s) - 1
+	for i, r := range s {
+		switch {
+		case r >= '0' && r <= '9':
+		case r >= 'A' && r <= 'Z':
+		case r >= 'a' && r <= 'z':
+		case r == '-' && i > 0 && i < n:
+		case r == '_':
+		case r == '*' && i == n:
+		default:
+			return false
+		}
+	}
+	return true
+}