// Copyright 2020 The prometheus-operator Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "context" "crypto/tls" "flag" "fmt" "log/slog" "net" "net/http" "os" "path/filepath" "time" "k8s.io/apiserver/pkg/server/dynamiccertificates" kflag "k8s.io/component-base/cli/flag" "github.com/prometheus-operator/prometheus-operator/pkg/operator" ) const ( defaultTLSDir = "/etc/tls/private" defaultTLSVersion = "VersionTLS13" ) func DefaultConfig(listenAddress string, enableTLS bool) Config { return Config{ ListenAddress: listenAddress, // Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go // standard library and golang.org/x/net are fully fixed. // Right now, it is possible for authenticated and unauthenticated users to // hold open HTTP2 connections and consume huge amounts of memory. // See: // * https://github.com/kubernetes/kubernetes/pull/121120 // * https://github.com/kubernetes/kubernetes/issues/121197 // * https://github.com/golang/go/issues/63417#issuecomment-1758858612 EnableHTTP2: false, TLSConfig: TLSConfig{ Enabled: enableTLS, CertFile: filepath.Join(defaultTLSDir, "tls.crt"), KeyFile: filepath.Join(defaultTLSDir, "tls.key"), ClientCAFile: filepath.Join(defaultTLSDir, "tls-ca.crt"), MinVersion: defaultTLSVersion, CipherSuites: operator.StringSet{}, ReloadInterval: time.Minute, }, } } func RegisterFlags(fs *flag.FlagSet, c *Config) { fs.StringVar(&c.ListenAddress, "web.listen-address", c.ListenAddress, "Address on which to expose metrics and web interface.") fs.BoolVar(&c.EnableHTTP2, "web.enable-http2", c.EnableHTTP2, "Enable HTTP2 connections.") fs.BoolVar(&c.TLSConfig.Enabled, "web.enable-tls", c.TLSConfig.Enabled, "Enable TLS for the web server.") fs.StringVar(&c.TLSConfig.CertFile, "web.cert-file", c.TLSConfig.CertFile, "Certificate file to be used for the web server.") fs.StringVar(&c.TLSConfig.KeyFile, "web.key-file", c.TLSConfig.KeyFile, "Private key matching the cert file to be used for the web server.") fs.StringVar(&c.TLSConfig.ClientCAFile, "web.client-ca-file", c.TLSConfig.ClientCAFile, "Client CA certificate file to be used for the web server.") fs.DurationVar(&c.TLSConfig.ReloadInterval, "web.tls-reload-interval", c.TLSConfig.ReloadInterval, "The interval at which to watch for TLS certificate changes, by default set to 1 minute. (default 1m0s).") fs.StringVar(&c.TLSConfig.MinVersion, "web.tls-min-version", c.TLSConfig.MinVersion, "Minimum TLS version supported. Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants.") fs.Var(&c.TLSConfig.CipherSuites, "web.tls-cipher-suites", "Comma-separated list of cipher suites for the server."+ " Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants)."+ "If omitted, the default Go cipher suites will be used. "+ "Note that TLS 1.3 ciphersuites are not configurable.") } // Config defines the web server configuration. type Config struct { ListenAddress string EnableHTTP2 bool TLSConfig TLSConfig } // TLSConfig defines the TLS settings of the web server. type TLSConfig struct { Enabled bool CertFile string KeyFile string ClientCAFile string MinVersion string CipherSuites operator.StringSet ReloadInterval time.Duration } // Convert returns a *tls.Config from the given TLSConfig. // It returns nil when TLS isn't enabled/configured. func (tc *TLSConfig) Convert(logger *slog.Logger) (*tls.Config, error) { if logger == nil { logger = slog.New(slog.DiscardHandler) } if !tc.Enabled { return nil, nil } if tc.CertFile == "" && tc.KeyFile == "" { if tc.ClientCAFile != "" { return nil, fmt.Errorf("server key and certificate must be provided when a client CA is configured") } // Disable TLS. logger.Warn("server key and certificate not provided, TLS disabled") return nil, nil } tlsCfg := &tls.Config{} version, err := kflag.TLSVersion(tc.MinVersion) if err != nil { return nil, fmt.Errorf("invalid TLS version: %w", err) } // Any older versions won't allow a secure connection. switch version { case tls.VersionTLS12: case tls.VersionTLS13: default: return nil, fmt.Errorf("TLS version %q isn't supported", tls.VersionName(version)) } tlsCfg.MinVersion = version cipherSuiteIDs, err := kflag.TLSCipherSuites(tc.CipherSuites.Slice()) if err != nil { return nil, fmt.Errorf("failed to convert TLS cipher suite name to ID: %w", err) } // A list of supported cipher suites for TLS versions up to TLS 1.2. // If CipherSuites is nil, a default list of secure cipher suites is used. // Note that TLS 1.3 ciphersuites are not configurable. tlsCfg.CipherSuites = cipherSuiteIDs if tc.ClientCAFile == "" { return tlsCfg, nil } info, err := os.Stat(tc.ClientCAFile) switch { case err != nil: logger.Warn("server TLS client verification disabled", "client_ca_file", tc.ClientCAFile, "err", err) case !info.Mode().IsRegular(): logger.Warn("server TLS client verification disabled", "client_ca_file", tc.ClientCAFile, "file_mode", info.Mode().String()) default: // The client CA content will be checked by the cert controller. tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert logger.Info("server TLS client verification enabled", "client_ca_file", tc.ClientCAFile) } return tlsCfg, nil } // Server is a web server. type Server struct { logger *slog.Logger listener net.Listener srv *http.Server runners []func(context.Context) cfg *Config } // NewServer initializes a web server with the given handler (typically an http.MuxServe). func NewServer(logger *slog.Logger, c *Config, handler http.Handler) (*Server, error) { listener, err := net.Listen("tcp", c.ListenAddress) if err != nil { return nil, err } tlsConfig, err := c.TLSConfig.Convert(logger) if err != nil { return nil, fmt.Errorf("failed to create TLS configuration: %w", err) } var runners []func(context.Context) if tlsConfig != nil { var ( certController *dynamiccertificates.DynamicServingCertificateController clientCA dynamiccertificates.CAContentProvider servingCert dynamiccertificates.CertKeyContentProvider ) if tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert { clientCA, err = dynamiccertificates.NewDynamicCAContentFromFile("clientCA", c.TLSConfig.ClientCAFile) if err != nil { return nil, fmt.Errorf("failed to load client CA certificate: %w", err) } if err = clientCA.(*dynamiccertificates.DynamicFileCAContent).RunOnce(context.Background()); err != nil { return nil, fmt.Errorf("failed to sync client CA certificate: %w", err) } runners = append(runners, func(ctx context.Context) { clientCA.(*dynamiccertificates.DynamicFileCAContent).Run(ctx, 1) }) } if c.TLSConfig.CertFile != "" && c.TLSConfig.KeyFile != "" { servingCert, err = dynamiccertificates.NewDynamicServingContentFromFiles("servingCert", c.TLSConfig.CertFile, c.TLSConfig.KeyFile) if err != nil { return nil, fmt.Errorf("failed to load serving certificate and key: %w", err) } if err = servingCert.(*dynamiccertificates.DynamicCertKeyPairContent).RunOnce(context.Background()); err != nil { return nil, fmt.Errorf("failed to sync serving certificate: %w", err) } runners = append(runners, func(ctx context.Context) { servingCert.(*dynamiccertificates.DynamicCertKeyPairContent).Run(ctx, 1) }) } certController = dynamiccertificates.NewDynamicServingCertificateController( tlsConfig, clientCA, servingCert, nil, nil, ) if clientCA != nil { clientCA.AddListener(certController) } if servingCert != nil { servingCert.AddListener(certController) } // Ensure that the configuration is valid. err = certController.RunOnce() if err != nil { return nil, fmt.Errorf("failed to sync certificates: %w", err) } runners = append(runners, func(ctx context.Context) { certController.Run(1, ctx.Done()) }) tlsConfig.GetConfigForClient = certController.GetConfigForClient listener = tls.NewListener(listener, tlsConfig) } srv := &http.Server{ Handler: handler, TLSConfig: tlsConfig, ReadHeaderTimeout: 30 * time.Second, ReadTimeout: 30 * time.Second, ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError), } if !c.EnableHTTP2 { srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) } return &Server{ logger: logger, srv: srv, listener: listener, runners: runners, cfg: c, }, nil } // Serve starts the web server. It will block until the server is shutted down // or an error occurs. func (s *Server) Serve(ctx context.Context) error { for _, r := range s.runners { go r(ctx) } if s.srv.TLSConfig == nil { s.logger.Info("starting insecure server", "address", s.listener.Addr().String()) } else { s.logger.Info("starting secure server", "address", s.listener.Addr().String(), "http2", s.cfg.EnableHTTP2) } if err := s.srv.Serve(s.listener); err != http.ErrServerClosed { return err } return nil } // Shutdown closes gracefully all active connections. func (s *Server) Shutdown(ctx context.Context) error { s.logger.Info("shutting down web server") return s.srv.Shutdown(ctx) }