// Copyright 2016 CoreOS, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package resource import ( "context" "crypto/tls" "crypto/x509" "encoding/hex" "encoding/pem" "errors" "io" "net" "net/http" "net/url" "strings" "time" ignerrors "github.com/coreos/ignition/v2/config/shared/errors" "github.com/coreos/ignition/v2/config/v3_6_experimental/types" "github.com/coreos/ignition/v2/internal/earlyrand" "github.com/coreos/ignition/v2/internal/log" "github.com/coreos/ignition/v2/internal/util" "github.com/coreos/ignition/v2/internal/version" "github.com/vincent-petithory/dataurl" "golang.org/x/net/http/httpproxy" ) const ( initialBackoff = 200 * time.Millisecond maxBackoff = 5 * time.Second defaultHttpResponseHeaderTimeout = 10 defaultHttpTotalTimeout = 0 ) var ( ErrTimeout = errors.New("unable to fetch resource in time") ErrPEMDecodeFailed = errors.New("unable to decode PEM block") ) // HttpClient is a simple wrapper around the Go HTTP client that standardizes // the process and logging of fetching payloads. type HttpClient struct { client *http.Client logger *log.Logger timeout time.Duration transport *http.Transport cas map[string][]byte } func (f *Fetcher) UpdateHttpTimeoutsAndCAs(timeouts types.Timeouts, cas []types.Resource, proxy types.Proxy) error { if f.client == nil { if err := f.newHttpClient(); err != nil { return err } } // Update timeouts responseHeader := defaultHttpResponseHeaderTimeout total := defaultHttpTotalTimeout if timeouts.HTTPResponseHeaders != nil { responseHeader = *timeouts.HTTPResponseHeaders } if timeouts.HTTPTotal != nil { total = *timeouts.HTTPTotal } f.client.client.Timeout = time.Duration(total) * time.Second f.client.timeout = f.client.client.Timeout f.client.transport.ResponseHeaderTimeout = time.Duration(responseHeader) * time.Second f.client.client.Transport = f.client.transport // Update proxy f.client.transport.Proxy = func(req *http.Request) (*url.URL, error) { return proxyFuncFromIgnitionConfig(proxy)(req.URL) } f.client.client.Transport = f.client.transport // Update CAs if len(cas) == 0 { return nil } pool, err := x509.SystemCertPool() if err != nil { f.Logger.Err("Unable to read system certificate pool: %s", err) return err } for _, ca := range cas { cablob, err := f.getCABlob(ca) if err != nil { return err } if err := f.parseCABundle(cablob, ca, pool); err != nil { f.Logger.Err("Unable to parse CA bundle: %s", err) return err } } f.client.transport.TLSClientConfig = &tls.Config{RootCAs: pool} return nil } // parseCABundle parses a CA bundle which includes multiple CAs. func (f *Fetcher) parseCABundle(cablob []byte, ca types.Resource, pool *x509.CertPool) error { for len(cablob) > 0 { block, rest := pem.Decode(cablob) if block == nil { f.Logger.Err("Unable to decode CA (%v)", ca.Source) return ErrPEMDecodeFailed } cert, err := x509.ParseCertificate(block.Bytes) if err != nil { f.Logger.Err("Unable to parse CA (%v): %s", ca.Source, err) return err } f.Logger.Info("Adding %q to list of CAs", cert.Subject.CommonName) pool.AddCert(cert) cablob = rest } return nil } func (f *Fetcher) getCABlob(ca types.Resource) ([]byte, error) { // this is also already checked at validation time if ca.Source == nil { f.Logger.Crit("invalid CA: %v", ignerrors.ErrSourceRequired) return nil, ignerrors.ErrSourceRequired } if blob, ok := f.client.cas[*ca.Source]; ok { return blob, nil } u, err := url.Parse(*ca.Source) if err != nil { f.Logger.Crit("Unable to parse CA URL: %s", err) return nil, err } hasher, err := util.GetHasher(ca.Verification) if err != nil { f.Logger.Crit("Unable to get hasher: %s", err) return nil, err } var expectedSum []byte if hasher != nil { // explicitly ignoring the error here because the config should already // be validated by this point _, expectedSumString, _ := util.HashParts(ca.Verification) expectedSum, err = hex.DecodeString(expectedSumString) if err != nil { f.Logger.Crit("Error parsing verification string %q: %v", expectedSumString, err) return nil, err } } var headers http.Header if len(ca.HTTPHeaders) > 0 { headers, err = ca.HTTPHeaders.Parse() if err != nil { return nil, err } } var compression string if ca.Compression != nil { compression = *ca.Compression } cablob, err := f.FetchToBuffer(*u, FetchOptions{ Hash: hasher, Headers: headers, ExpectedSum: expectedSum, Compression: compression, }) if err != nil { f.Logger.Err("Unable to fetch CA (%s): %s", u, err) return nil, err } f.client.cas[*ca.Source] = cablob return cablob, nil } // RewriteCAsWithDataUrls will modify the passed in slice of CA references to // contain the actual CA file via a dataurl in their source field. func (f *Fetcher) RewriteCAsWithDataUrls(cas []types.Resource) error { for i, ca := range cas { blob, err := f.getCABlob(ca) if err != nil { return err } // Clean HTTP headers cas[i].HTTPHeaders = nil // the rewrite wipes the compression cas[i].Compression = nil encoded := dataurl.EncodeBytes(blob) cas[i].Source = &encoded } return nil } // DefaultHTTPClient builds the default `http.client` for Ignition. func defaultHTTPClient() (*http.Client, error) { urand, err := earlyrand.UrandomReader() if err != nil { return nil, err } tlsConfig := tls.Config{ Rand: urand, } transport := http.Transport{ ResponseHeaderTimeout: time.Duration(defaultHttpResponseHeaderTimeout) * time.Second, Dial: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, Resolver: &net.Resolver{ PreferGo: true, }, }).Dial, TLSClientConfig: &tlsConfig, TLSHandshakeTimeout: 10 * time.Second, } client := http.Client{ Transport: &transport, } return &client, nil } // newHttpClient populates the fetcher with the default HTTP client. func (f *Fetcher) newHttpClient() error { defaultClient, err := defaultHTTPClient() if err != nil { return err } f.client = &HttpClient{ client: defaultClient, logger: f.Logger, timeout: time.Duration(defaultHttpTotalTimeout) * time.Second, transport: defaultClient.Transport.(*http.Transport), cas: make(map[string][]byte), } return nil } func shouldRetryHttp(statusCode int, opts FetchOptions) bool { // we always retry 500+ if statusCode >= 500 { return true } for _, retryCode := range opts.RetryCodes { if statusCode == retryCode { return true } } return false } // httpReaderWithHeader performs an HTTP request on the provided URL with the // provided request header & method and returns the response body Reader, HTTP // status code, a cancel function for the result's context, and error (if any). // By default, User-Agent is added to the header but this can be overridden. func (c HttpClient) httpReaderWithHeader(opts FetchOptions, url string) (io.ReadCloser, int, context.CancelFunc, error) { if opts.HTTPVerb == "" { opts.HTTPVerb = "GET" } req, err := http.NewRequest(opts.HTTPVerb, url, nil) if err != nil { return nil, 0, nil, err } req.Header.Set("User-Agent", "Ignition/"+version.Raw) for key, values := range opts.Headers { req.Header.Del(key) for _, value := range values { req.Header.Add(key, value) } } ctx, cancelFn := context.WithCancel(context.Background()) if c.timeout != 0 { cancelFn() ctx, cancelFn = context.WithTimeout(context.Background(), c.timeout) } duration := initialBackoff for attempt := 1; ; attempt++ { c.logger.Info("%s %s: attempt #%d", opts.HTTPVerb, url, attempt) resp, err := c.client.Do(req.WithContext(ctx)) if err == nil { c.logger.Info("%s result: %s", opts.HTTPVerb, http.StatusText(resp.StatusCode)) if !shouldRetryHttp(resp.StatusCode, opts) { return resp.Body, resp.StatusCode, cancelFn, nil } _ = resp.Body.Close() } else { c.logger.Info("%s error: %v", opts.HTTPVerb, err) } // Wait before next attempt or exit if we timeout while waiting select { case <-time.After(duration): case <-ctx.Done(): return nil, 0, cancelFn, ErrTimeout } duration = duration * 2 if duration > maxBackoff { duration = maxBackoff } } } func proxyFuncFromIgnitionConfig(proxy types.Proxy) func(*url.URL) (*url.URL, error) { noProxy := translateNoProxySliceToString(proxy.NoProxy) if proxy.HTTPProxy == nil { proxy.HTTPProxy = new(string) } if proxy.HTTPSProxy == nil { proxy.HTTPSProxy = new(string) } cfg := &httpproxy.Config{ HTTPProxy: *proxy.HTTPProxy, HTTPSProxy: *proxy.HTTPSProxy, NoProxy: noProxy, } return cfg.ProxyFunc() } func translateNoProxySliceToString(items []types.NoProxyItem) string { newItems := make([]string, len(items)) for i, o := range items { newItems[i] = string(o) } return strings.Join(newItems, ",") }