From cc2d511de317d24d63e4fac811dc555cdd1833fc Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 23 Nov 2023 12:25:31 -0600 Subject: [PATCH] Periodically prepare a combined custom + system CA bundle Both XRootD and Prometheus can't use our custom transport object, which combines the system CA pool with the custom CA. This causes various failures unless you disable TLS validation (which we want to avoid doing). This adds a helper function that periodically (re)creates the CA bundle. These bundles are made available to other functions at runtime. NOTE: Neither Mac nor Windows have a standard system bundle. On those platforms, we only create the bundle if the custom CA is present. --- utils/ca_utils.go | 225 ++++++++++++++++++++++++++++++++++++++++ web_ui/prometheus.go | 17 ++- xrootd/xrootd_config.go | 11 ++ 3 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 utils/ca_utils.go diff --git a/utils/ca_utils.go b/utils/ca_utils.go new file mode 100644 index 000000000..58fd5aae2 --- /dev/null +++ b/utils/ca_utils.go @@ -0,0 +1,225 @@ +/*************************************************************** + * + * Copyright (C) 2023, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +package utils + +import ( + "crypto/x509" + "encoding/pem" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/pelicanplatform/pelican/param" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +// Write out all the trusted CAs as a CA bundle on disk. This is useful +// for components that do not use go's trusted CA store +func WriteCABundle(filename string) (int, error) { + roots, err := loadSystemRoots() + if err != nil { + return -1, errors.Wrap(err, "Unable to write CA bundle due to failure when loading system trust roots") + } + + // Append in any custom CAs we might have + caFile := param.Server_TLSCACertificateFile.GetString() + pemContents, err := os.ReadFile(caFile) + if err == nil { + roots = append(roots, getCertsFromPEM(pemContents)...) + } + + if len(roots) == 0 { + return 0, nil + } + + dir := filepath.Dir(filename) + base := filepath.Base(filename) + file, err := os.CreateTemp(dir, base) + if err != nil { + return -1, errors.Wrap(err, "Unable to create CA bundle temporary file") + } + defer file.Close() + if err = os.Chmod(file.Name(), 0644); err != nil { + return -1, errors.Wrap(err, "Failed to chmod CA bundle temporary file") + } + + for _, root := range roots { + if err = pem.Encode(file, &pem.Block{Type: "CERTIFICATE", Bytes: root.Raw}); err != nil { + return -1, errors.Wrap(err, "Failed to write CA into bundle") + } + } + + if err := os.Rename(file.Name(), filename); err != nil { + return -1, errors.Wrapf(err, "Failed to move temporary CA bundle to final location (%v)", filename) + } + + return len(roots), nil +} + +// Periodically write out the system CAs, updating them if the system updates. +// Returns an error if the first attempt at writing fails. Otherwise, it will +// launch a goroutine and update the entire CA bundle every specified duration. +// +// If we're on a platform (Mac, Windows) that does not provide a CA bundle, we return +// a count of 0 and do not launch the go routine. +func PeriodicWriteCABundle(filename string, sleepTime time.Duration) (count int, err error) { + count, err = WriteCABundle(filename) + if err != nil || count == 0 { + return + } + + go func() { + time.Sleep(sleepTime) + _, err := WriteCABundle(filename) + if err != nil { + log.Warningln("Failure during periodic CA bundle update:", err) + } + }() + + return +} + +// NOTE: Code below is taken from src/crypto/x509/root_unix.go in the go runtime. Since the +// runtime is BSD-licensed, it is compatible with its inclusion in Pelican +const ( + certFileEnv = "SSL_CERT_FILE" + certDirEnv = "SSL_CERT_DIR" +) + +var certFiles = []string{ + "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc. + "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6 + "/etc/ssl/ca-bundle.pem", // OpenSUSE + "/etc/pki/tls/cacert.pem", // OpenELEC + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7 + "/etc/ssl/cert.pem", // Alpine Linux +} + +var certDirectories = []string{ + "/etc/ssl/certs", // SLES10/SLES11, https://golang.org/issue/12139 + "/etc/pki/tls/certs", // Fedora/RHEL +} + +func getCertsFromPEM(pemCerts []byte) []*x509.Certificate { + result := make([]*x509.Certificate, 0) + for len(pemCerts) > 0 { + var block *pem.Block + block, pemCerts = pem.Decode(pemCerts) + if block == nil { + break + } + if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + continue + } + certBytes := block.Bytes + cert, err := x509.ParseCertificate(certBytes) + if err != nil { + continue + } + result = append(result, cert) + } + return result +} + +func loadSystemRoots() ([]*x509.Certificate, error) { + // The code below only works on Linux; other platforms require syscalls + // On those, we simply return no system CAs. + roots := make([]*x509.Certificate, 0) + if os := runtime.GOOS; os != "linux" { + return roots, nil + } + + files := certFiles + if f := os.Getenv(certFileEnv); f != "" { + files = []string{f} + } + + var firstErr error + for _, file := range files { + pemCerts, err := os.ReadFile(file) + if err == nil { + roots = append(roots, getCertsFromPEM(pemCerts)...) + break + } + if firstErr == nil && !os.IsNotExist(err) { + firstErr = err + } + } + + dirs := certDirectories + if d := os.Getenv(certDirEnv); d != "" { + // OpenSSL and BoringSSL both use ":" as the SSL_CERT_DIR separator. + // See: + // * https://golang.org/issue/35325 + // * https://www.openssl.org/docs/man1.0.2/man1/c_rehash.html + dirs = strings.Split(d, ":") + } + + for _, directory := range dirs { + fis, err := readUniqueDirectoryEntries(directory) + if err != nil { + if firstErr == nil && !os.IsNotExist(err) { + firstErr = err + } + continue + } + for _, fi := range fis { + data, err := os.ReadFile(directory + "/" + fi.Name()) + if err == nil { + roots = append(roots, getCertsFromPEM(data)...) + } + } + } + + if len(roots) > 0 || firstErr == nil { + return roots, nil + } + + return nil, firstErr +} + +// readUniqueDirectoryEntries is like os.ReadDir but omits +// symlinks that point within the directory. +func readUniqueDirectoryEntries(dir string) ([]fs.DirEntry, error) { + files, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + uniq := files[:0] + for _, f := range files { + if !isSameDirSymlink(f, dir) { + uniq = append(uniq, f) + } + } + return uniq, nil +} + +// isSameDirSymlink reports whether a file in a dir is a symlink with a +// target not containing a slash. +func isSameDirSymlink(f fs.DirEntry, dir string) bool { + if f.Type()&fs.ModeSymlink == 0 { + return false + } + target, err := os.Readlink(filepath.Join(dir, f.Name())) + return err == nil && !strings.Contains(target, "/") +} diff --git a/web_ui/prometheus.go b/web_ui/prometheus.go index 2292d65ef..9332915f1 100644 --- a/web_ui/prometheus.go +++ b/web_ui/prometheus.go @@ -17,13 +17,13 @@ package web_ui import ( "context" - "errors" "fmt" "math" "net/http" "net/url" "os" "os/signal" + "path/filepath" "strings" "sync" "syscall" @@ -38,6 +38,8 @@ import ( "github.com/oklog/run" "github.com/pelicanplatform/pelican/director" "github.com/pelicanplatform/pelican/param" + "github.com/pelicanplatform/pelican/utils" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/version" @@ -176,6 +178,15 @@ func configDirectorPromScraper() (*config.ScrapeConfig, error) { scrapeConfig := config.DefaultScrapeConfig scrapeConfig.JobName = "origins" scrapeConfig.Scheme = "https" + + // This will cause the director to maintain a CA bundle, including the custom CA, at + // the given location. Makes up for the fact we can't provide Prometheus with a transport + caBundle := filepath.Join(param.Monitoring_DataLocation.GetString(), "ca-bundle.crt") + caCount, err := utils.PeriodicWriteCABundle(caBundle, 2*time.Minute) + if err != nil { + return nil, errors.Wrap(err, "Unable to generate CA bundle for prometheus") + } + scraperHttpClientConfig := common_config.HTTPClientConfig{ TLSConfig: common_config.TLSConfig{ // For the scraper to origins' metrics, we get TLSSkipVerify from config @@ -188,6 +199,10 @@ func configDirectorPromScraper() (*config.ScrapeConfig, error) { Credentials: common_config.Secret(scraperToken), }, } + if caCount > 0 { + scraperHttpClientConfig.TLSConfig.CAFile = caBundle + } + scrapeConfig.HTTPClientConfig = scraperHttpClientConfig scrapeConfig.ServiceDiscoveryConfigs = make([]discovery.Config, 1) sdHttpClientConfig := common_config.HTTPClientConfig{ diff --git a/xrootd/xrootd_config.go b/xrootd/xrootd_config.go index d6877a609..a46434ddd 100644 --- a/xrootd/xrootd_config.go +++ b/xrootd/xrootd_config.go @@ -13,12 +13,14 @@ import ( "strings" "sync" "text/template" + "time" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/director" "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/origin_ui" "github.com/pelicanplatform/pelican/param" + "github.com/pelicanplatform/pelican/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -350,6 +352,15 @@ func ConfigXrootd(origin bool) (string, error) { return "", err } + runtimeCAs := filepath.Join(param.Xrootd_RunLocation.GetString(), "ca-bundle.crt") + caCount, err := utils.PeriodicWriteCABundle(runtimeCAs, 2*time.Minute) + if err != nil { + return "", errors.Wrap(err, "Failed to setup the runtime CA bundle") + } + if caCount > 0 { + xrdConfig.Server.TLSCACertificateFile = runtimeCAs + } + if origin { if xrdConfig.Origin.Multiuser { ok, err := config.HasMultiuserCaps()