diff --git a/cmd/discovery/main.go b/cmd/discovery/main.go index 9aea6001d06..dd2dd9cff56 100644 --- a/cmd/discovery/main.go +++ b/cmd/discovery/main.go @@ -18,6 +18,7 @@ import ( "net/http" _ "net/http/pprof" "os" + "strconv" "time" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" @@ -33,12 +34,14 @@ import ( var ( printVersion bool port int + proxyPort int ) func init() { flag.BoolVar(&printVersion, "V", false, "Show version and quit") flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.IntVar(&port, "port", 10261, "The port that the tidb discovery's http service runs on (default 10261)") + flag.IntVar(&proxyPort, "proxy-port", 10262, "The port that the tidb discovery's proxy service runs on (default 10262)") flag.Parse() } @@ -69,8 +72,22 @@ func main() { klog.Fatalf("failed to get kubernetes Clientset: %v", err) } + tcName := os.Getenv("TC_NAME") + if len(tcName) < 1 { + klog.Fatal("ENV TC_NAME is not set") + } + tcTls := false + tlsEnabled := os.Getenv("TC_TLS_ENABLED") + if tlsEnabled == strconv.FormatBool(true) { + tcTls = true + } + go wait.Forever(func() { server.StartServer(cli, kubeCli, port) }, 5*time.Second) + go wait.Forever(func() { + server.StartProxyServer(tcName, tcTls, proxyPort) + }, 5*time.Second) + klog.Fatal(http.ListenAndServe(":6060", nil)) } diff --git a/pkg/discovery/server/proxy.go b/pkg/discovery/server/proxy.go new file mode 100644 index 00000000000..e07eec0cfd2 --- /dev/null +++ b/pkg/discovery/server/proxy.go @@ -0,0 +1,110 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" + "net/url" + "strings" + + "github.com/pingcap/tidb-operator/pkg/manager/member" + "k8s.io/klog" +) + +func buildUrl(tcName string, tlsEnabled bool) (*url.URL, error) { + url := &url.URL{ + Host: fmt.Sprintf("%s-pd:2379", tcName), + Scheme: "http", + } + + if tlsEnabled { + url.Scheme = "https" + } + return url, nil +} + +func buildProxy(tcName string, tlsEnabled bool) (*httputil.ReverseProxy, error) { + url, err := buildUrl(tcName, tlsEnabled) + if err != nil { + klog.Error(err) + return nil, err + } + proxy := httputil.NewSingleHostReverseProxy(url) + if tlsEnabled { + // load crt and key + certPath := fmt.Sprintf("%s/tls.crt", member.PdTlsCertPath) + keyPath := fmt.Sprintf("%s/tls.key", member.PdTlsCertPath) + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + klog.Error(err) + return nil, err + } + // load ca + rootCAs := x509.NewCertPool() + caPath := fmt.Sprintf("%s/ca.crt", member.PdTlsCertPath) + caByte, err := ioutil.ReadFile(caPath) + if err != nil { + klog.Error(err) + return nil, err + } + rootCAs.AppendCertsFromPEM(caByte) + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: rootCAs, + } + proxy.Transport = &http.Transport{TLSClientConfig: tlsConfig} + } + director := proxy.Director + proxy.Director = func(req *http.Request) { + if strings.HasPrefix(req.RequestURI, "/dashboard") { + director(req) + req.Host = req.URL.Host + } + } + return proxy, nil +} + +type handler struct { + tcName string + namespace string + tcTlsEnabled bool +} + +func NewHandler(tcName string, tcTlsEnabled bool) *handler { + return &handler{ + tcName: tcName, + tcTlsEnabled: tcTlsEnabled, + } +} + +func (handler *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + proxy, err := buildProxy(handler.tcName, handler.tcTlsEnabled) + if err != nil { + msg := fmt.Sprintf("Error Happed, err:%v", err) + w.Write([]byte(msg)) + return + } + proxy.ServeHTTP(w, req) +} + +func StartProxyServer(tcName string, tcTlsEnabled bool, port int) { + handler := NewHandler(tcName, tcTlsEnabled) + klog.Infof("start proxy-server") + klog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), handler)) +} diff --git a/pkg/manager/member/tidb_discovery_manager.go b/pkg/manager/member/tidb_discovery_manager.go index 5dabd068491..af50f638d87 100644 --- a/pkg/manager/member/tidb_discovery_manager.go +++ b/pkg/manager/member/tidb_discovery_manager.go @@ -15,10 +15,12 @@ package member import ( "encoding/json" + "strconv" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -26,6 +28,10 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +const ( + PdTlsCertPath = "/var/lib/pd-tls" +) + type TidbDiscoveryManager interface { Reconcile(tc *v1alpha1.TidbCluster) error } @@ -105,12 +111,20 @@ func getTidbDiscoveryService(tc *v1alpha1.TidbCluster, deploy *appsv1.Deployment ObjectMeta: meta, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{{ - Name: "discovery", - Port: 10261, - TargetPort: intstr.FromInt(10261), - Protocol: corev1.ProtocolTCP, - }}, + Ports: []corev1.ServicePort{ + { + Name: "discovery", + Port: 10261, + TargetPort: intstr.FromInt(10261), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "proxy", + Port: 10262, + TargetPort: intstr.FromInt(10262), + Protocol: corev1.ProtocolTCP, + }, + }, Selector: deploy.Spec.Template.Labels, }, } @@ -150,12 +164,39 @@ func getTidbDiscoveryDeployment(tc *v1alpha1.TidbCluster) (*appsv1.Deployment, e Name: "TZ", Value: tc.Timezone(), }, + { + Name: "TC_NAME", + Value: tc.Name, + }, }, }}, }, }, }, } + if tc.IsTLSClusterEnabled() { + d.Spec.Template.Spec.Volumes = []corev1.Volume{ + { + Name: "pd-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: util.ClusterTLSSecretName(tc.Name, label.PDLabelVal), + }, + }, + }, + } + d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + Name: "pd-tls", + ReadOnly: true, + MountPath: PdTlsCertPath, + }, + } + d.Spec.Template.Spec.Containers[0].Env = append(d.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{ + Name: "TC_TLS_ENABLED", + Value: strconv.FormatBool(true), + }) + } b, err := json.Marshal(d.Spec.Template.Spec) if err != nil { return nil, err