diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go
index 46e45c64652d..b11524fb3d5d 100644
--- a/modules/caddyhttp/reverseproxy/upstreams.go
+++ b/modules/caddyhttp/reverseproxy/upstreams.go
@@ -313,8 +313,8 @@ func (au *AUpstreams) Provision(ctx caddy.Context) error {
 func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
-	resolveIpv4 := au.Versions == nil || au.Versions.IPv4 == nil || *au.Versions.IPv4
-	resolveIpv6 := au.Versions == nil || au.Versions.IPv6 == nil || *au.Versions.IPv6
+	resolveIpv4 := au.Versions == nil || (au.Versions.IPv4 == nil && au.Versions.IPv6 == nil) || (au.Versions.IPv6 != nil && *au.Versions.IPv4)
+	resolveIpv6 := au.Versions == nil || (au.Versions.IPv6 == nil && au.Versions.IPv4 == nil) || (au.Versions.IPv6 != nil && *au.Versions.IPv6)
 
 	// Map ipVersion early, so we can use it as part of the cache-key.
 	// This should be fairly inexpensive and comes and the upside of