Skip to content

Commit

Permalink
Add proxy server for Dashboard (#2684)
Browse files Browse the repository at this point in the history
  • Loading branch information
Yisaer authored Jun 12, 2020
1 parent bc93aeb commit 40c2bc9
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 6 deletions.
17 changes: 17 additions & 0 deletions cmd/discovery/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"net/http"
_ "net/http/pprof"
"os"
"strconv"
"time"

"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
Expand All @@ -33,12 +34,14 @@ import (
var (
printVersion bool
port int
proxyPort int
)

func init() {
flag.BoolVar(&printVersion, "V", false, "Show version and quit")
flag.BoolVar(&printVersion, "version", false, "Show version and quit")
flag.IntVar(&port, "port", 10261, "The port that the tidb discovery's http service runs on (default 10261)")
flag.IntVar(&proxyPort, "proxy-port", 10262, "The port that the tidb discovery's proxy service runs on (default 10262)")
flag.Parse()
}

Expand Down Expand Up @@ -69,8 +72,22 @@ func main() {
klog.Fatalf("failed to get kubernetes Clientset: %v", err)
}

tcName := os.Getenv("TC_NAME")
if len(tcName) < 1 {
klog.Fatal("ENV TC_NAME is not set")
}
tcTls := false
tlsEnabled := os.Getenv("TC_TLS_ENABLED")
if tlsEnabled == strconv.FormatBool(true) {
tcTls = true
}

go wait.Forever(func() {
server.StartServer(cli, kubeCli, port)
}, 5*time.Second)
go wait.Forever(func() {
server.StartProxyServer(tcName, tcTls, proxyPort)
}, 5*time.Second)

klog.Fatal(http.ListenAndServe(":6060", nil))
}
110 changes: 110 additions & 0 deletions pkg/discovery/server/proxy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net/http"
"net/http/httputil"
"net/url"
"strings"

"github.com/pingcap/tidb-operator/pkg/manager/member"
"k8s.io/klog"
)

func buildUrl(tcName string, tlsEnabled bool) (*url.URL, error) {
url := &url.URL{
Host: fmt.Sprintf("%s-pd:2379", tcName),
Scheme: "http",
}

if tlsEnabled {
url.Scheme = "https"
}
return url, nil
}

func buildProxy(tcName string, tlsEnabled bool) (*httputil.ReverseProxy, error) {
url, err := buildUrl(tcName, tlsEnabled)
if err != nil {
klog.Error(err)
return nil, err
}
proxy := httputil.NewSingleHostReverseProxy(url)
if tlsEnabled {
// load crt and key
certPath := fmt.Sprintf("%s/tls.crt", member.PdTlsCertPath)
keyPath := fmt.Sprintf("%s/tls.key", member.PdTlsCertPath)
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
klog.Error(err)
return nil, err
}
// load ca
rootCAs := x509.NewCertPool()
caPath := fmt.Sprintf("%s/ca.crt", member.PdTlsCertPath)
caByte, err := ioutil.ReadFile(caPath)
if err != nil {
klog.Error(err)
return nil, err
}
rootCAs.AppendCertsFromPEM(caByte)
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: rootCAs,
}
proxy.Transport = &http.Transport{TLSClientConfig: tlsConfig}
}
director := proxy.Director
proxy.Director = func(req *http.Request) {
if strings.HasPrefix(req.RequestURI, "/dashboard") {
director(req)
req.Host = req.URL.Host
}
}
return proxy, nil
}

type handler struct {
tcName string
namespace string
tcTlsEnabled bool
}

func NewHandler(tcName string, tcTlsEnabled bool) *handler {
return &handler{
tcName: tcName,
tcTlsEnabled: tcTlsEnabled,
}
}

func (handler *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
proxy, err := buildProxy(handler.tcName, handler.tcTlsEnabled)
if err != nil {
msg := fmt.Sprintf("Error Happed, err:%v", err)
w.Write([]byte(msg))
return
}
proxy.ServeHTTP(w, req)
}

func StartProxyServer(tcName string, tcTlsEnabled bool, port int) {
handler := NewHandler(tcName, tcTlsEnabled)
klog.Infof("start proxy-server")
klog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), handler))
}
53 changes: 47 additions & 6 deletions pkg/manager/member/tidb_discovery_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,23 @@ package member

import (
"encoding/json"
"strconv"

"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/pkg/util"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)

const (
PdTlsCertPath = "/var/lib/pd-tls"
)

type TidbDiscoveryManager interface {
Reconcile(tc *v1alpha1.TidbCluster) error
}
Expand Down Expand Up @@ -105,12 +111,20 @@ func getTidbDiscoveryService(tc *v1alpha1.TidbCluster, deploy *appsv1.Deployment
ObjectMeta: meta,
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Ports: []corev1.ServicePort{{
Name: "discovery",
Port: 10261,
TargetPort: intstr.FromInt(10261),
Protocol: corev1.ProtocolTCP,
}},
Ports: []corev1.ServicePort{
{
Name: "discovery",
Port: 10261,
TargetPort: intstr.FromInt(10261),
Protocol: corev1.ProtocolTCP,
},
{
Name: "proxy",
Port: 10262,
TargetPort: intstr.FromInt(10262),
Protocol: corev1.ProtocolTCP,
},
},
Selector: deploy.Spec.Template.Labels,
},
}
Expand Down Expand Up @@ -150,12 +164,39 @@ func getTidbDiscoveryDeployment(tc *v1alpha1.TidbCluster) (*appsv1.Deployment, e
Name: "TZ",
Value: tc.Timezone(),
},
{
Name: "TC_NAME",
Value: tc.Name,
},
},
}},
},
},
},
}
if tc.IsTLSClusterEnabled() {
d.Spec.Template.Spec.Volumes = []corev1.Volume{
{
Name: "pd-tls",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: util.ClusterTLSSecretName(tc.Name, label.PDLabelVal),
},
},
},
}
d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
{
Name: "pd-tls",
ReadOnly: true,
MountPath: PdTlsCertPath,
},
}
d.Spec.Template.Spec.Containers[0].Env = append(d.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
Name: "TC_TLS_ENABLED",
Value: strconv.FormatBool(true),
})
}
b, err := json.Marshal(d.Spec.Template.Spec)
if err != nil {
return nil, err
Expand Down

0 comments on commit 40c2bc9

Please # to comment.