From 8cd259e8ae60011cab6ad5f053e8343486d49e83 Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 13 Mar 2024 18:08:22 -0400 Subject: [PATCH] journal client: force using http1.1 when fetching fragments from cloud storage We've seen some unfortunate issues with the http2 client, where certain types of transport failures can cause a connection to be unusable, and yet the client will still re-use it for new http2 streams. Disabling http2 is intended to be a short term fix, until the root cause can be identified and fixed. --- broker/client/reader.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/broker/client/reader.go b/broker/client/reader.go index fc1e5583..0a558835 100644 --- a/broker/client/reader.go +++ b/broker/client/reader.go @@ -3,6 +3,7 @@ package client import ( "bufio" "context" + "crypto/tls" "errors" "fmt" "io" @@ -389,6 +390,21 @@ func fragmentLabels(fragment pb.Fragment) prometheus.Labels { } } +// newHttpClient returns an http client for readers to use for fetching fragments. +// It disables http2 because we've observed some rather horrific behavior from http2 +// lately. When there's an error with the underlying transport, the http2 client can still +// use it for additional streams, creating the potential for connection failures to cause +// more widespread errors for other requests to the same host. Falling back to http 1.1 +// is intended to be a short term workaround. +func newHttpClient() *http.Client { + var transport = http.DefaultTransport.(*http.Transport).Clone() + // Recommended here: https://pkg.go.dev/net/http#hdr-HTTP_2 + transport.TLSNextProto = map[string]func(authority string, c *tls.Conn) http.RoundTripper{} + return &http.Client{ + Transport: transport, + } +} + var ( // Map common broker error statuses into named errors. ErrInsufficientJournalBrokers = errors.New(pb.Status_INSUFFICIENT_JOURNAL_BROKERS.String()) @@ -412,5 +428,5 @@ var ( ErrDidNotReadExpectedEOF = errors.New("did not read EOF at expected Fragment.End") // httpClient is the http.Client used by OpenFragmentURL - httpClient = http.DefaultClient + httpClient = newHttpClient() )