From 6baf48e2016cbdba1cbbf80c3b7419528cfaf418 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 8 Dec 2014 18:05:29 +0000 Subject: [PATCH 01/26] compiles against k8s v0.5x --- Godeps/.gitignore | 1 + Godeps/Godeps.json | 143 ++++---- Makefile | 3 +- controller-manager/controller-manager.go | 6 +- executor/executor.go | 58 ++-- executor/server.go | 97 +++++- hack/patches/k8s---issue77.patch | 28 +- kubernetes-executor/main.go | 58 +++- kubernetes-executor/plugins.go | 22 ++ kubernetes-mesos/main.go | 203 ++++++++---- master/handlers.go | 50 +++ master/master.go | 400 ++++++++++++++++++++--- master/publish.go | 138 ++++++++ scheduler/cloud.go | 7 + scheduler/cloud_registry.go | 17 +- scheduler/offers.go | 4 +- scheduler/pod_task.go | 8 +- scheduler/scheduler.go | 66 ++-- service/endpoints_controller.go | 76 +++-- 19 files changed, 1075 insertions(+), 310 deletions(-) create mode 100644 Godeps/.gitignore create mode 100644 kubernetes-executor/plugins.go create mode 100644 master/handlers.go create mode 100644 master/publish.go diff --git a/Godeps/.gitignore b/Godeps/.gitignore new file mode 100644 index 00000000..ece63be6 --- /dev/null +++ b/Godeps/.gitignore @@ -0,0 +1 @@ +_workspace diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index afb40fc9..1fc9cdd4 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -27,166 +27,171 @@ }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/api", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/client", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/constraint", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/conversion", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/health", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/httplog", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/labels", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/master", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/binding", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/controller", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/endpoint", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/etcd", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/minion", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/pod", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/service", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/service", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/tools", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/util", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/version", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/volume", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/watch", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler", - "Comment": "v0.4.4", - "Rev": "efd443bcd6f005566daa85da0a5f0b633b40d4e3" + "Comment": "v0.5.4", + "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" }, { "ImportPath": "github.com/coreos/go-etcd/etcd", "Comment": "v0.2.0-rc1-120-g23142f6", "Rev": "23142f6773a676cc2cae8dd0cb90b2ea761c853f" }, - { - "ImportPath": "github.com/elazarl/go-bindata-assetfs", - "Rev": "ae4665cf2d188c65764c73fe4af5378acc549510" - }, + { + "ImportPath": "github.com/elazarl/go-bindata-assetfs", + "Rev": "ae4665cf2d188c65764c73fe4af5378acc549510" + }, + { + "ImportPath": "github.com/emicklei/go-restful", + "Comment": "v1.1.2-34-gcb26ade", + "Rev": "cb26adeb9644200cb4ec7b32be31e024696e8d00" + }, { "ImportPath": "github.com/fsouza/go-dockerclient", - "Comment": "0.2.1-241-g0dbb508", - "Rev": "0dbb508e94dd899a6743d035d8f249c7634d26da" + "Comment": "0.2.1-267-g15d2c6e", + "Rev": "15d2c6e3eb670c545d0af0604d7f9aff3871af04" }, { "ImportPath": "github.com/golang/glog", - "Rev": "d1c4472bf2efd3826f2b5bdcc02d8416798d678c" + "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" }, { "ImportPath": "github.com/google/cadvisor/info", - "Comment": "0.4.0", - "Rev": "5a6d06c02600b1e57e55a9d9f71dbac1bfc9fe6c" + "Comment": "0.5.0", + "Rev": "8c4f650e62f096710da794e536de86e34447fca9" }, { "ImportPath": "github.com/mesos/mesos-go/mesos", @@ -196,6 +201,14 @@ "ImportPath": "github.com/skratchdot/open-golang/open", "Rev": "ba570a111973b539baf23c918213059543b5bb6e" }, + { + "ImportPath": "github.com/spf13/cobra", + "Rev": "b1e90a7943957b51bb96a13b44b844475bcf95c0" + }, + { + "ImportPath": "github.com/spf13/pflag", + "Rev": "463bdc838f2b35e9307e91d480878bda5fff7232" + }, { "ImportPath": "gopkg.in/v1/yaml", "Rev": "1b9791953ba4027efaeb728c7355e542a203be5e" diff --git a/Makefile b/Makefile index 62ab5ad1..905faf00 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,8 @@ KUBE_GO_PACKAGE ?= github.com/GoogleCloudPlatform/kubernetes K8S_CMD := \ ${KUBE_GO_PACKAGE}/cmd/kubecfg \ - ${KUBE_GO_PACKAGE}/cmd/proxy + ${KUBE_GO_PACKAGE}/cmd/kubectl \ + ${KUBE_GO_PACKAGE}/cmd/kube-proxy FRAMEWORK_CMD := \ github.com/mesosphere/kubernetes-mesos/controller-manager \ github.com/mesosphere/kubernetes-mesos/kubernetes-mesos \ diff --git a/controller-manager/controller-manager.go b/controller-manager/controller-manager.go index 0eeb65b2..93a1610c 100644 --- a/controller-manager/controller-manager.go +++ b/controller-manager/controller-manager.go @@ -33,7 +33,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/controller" _ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz" - masterPkg "github.com/GoogleCloudPlatform/kubernetes/pkg/master" + "github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports" kendpoint "github.com/GoogleCloudPlatform/kubernetes/pkg/service" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag" @@ -43,10 +43,10 @@ import ( ) var ( - port = flag.Int("port", masterPkg.ControllerManagerPort, "The port that the controller-manager's http service runs on") + port = flag.Int("port", ports.ControllerManagerPort, "The port that the controller-manager's http service runs on") address = util.IP(net.ParseIP("127.0.0.1")) - useHostPortEndpoints = flag.Bool("host_port_endpoints", true, "Map service endpoints to hostIP:hostPort instead of podIP:containerPort. Default true.") clientConfig = &client.Config{} + useHostPortEndpoints = flag.Bool("host_port_endpoints", true, "Map service endpoints to hostIP:hostPort instead of podIP:containerPort. Default true.") ) func init() { diff --git a/executor/executor.go b/executor/executor.go index 6b330d60..4e22f705 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -19,8 +19,8 @@ const ( ) type kuberTask struct { - mesosTaskInfo *mesos.TaskInfo - containerManifest *api.ContainerManifest + mesosTaskInfo *mesos.TaskInfo + podName string } // KubernetesExecutor is an mesos executor that runs pods @@ -31,9 +31,9 @@ type KubernetesExecutor struct { driver mesos.ExecutorDriver registered bool tasks map[string]*kuberTask - pods map[string]*kubelet.Pod + pods map[string]*api.BoundPod lock sync.RWMutex - namespace string + sourcename string } // New creates a new kubernetes executor. @@ -44,8 +44,8 @@ func New(driver mesos.ExecutorDriver, kl *kubelet.Kubelet, ch chan<- interface{} driver: driver, registered: false, tasks: make(map[string]*kuberTask), - pods: make(map[string]*kubelet.Pod), - namespace: ns, + pods: make(map[string]*api.BoundPod), + sourcename: ns, } } @@ -91,36 +91,32 @@ func (k *KubernetesExecutor) LaunchTask(driver mesos.ExecutorDriver, taskInfo *m // may be duplicated messages or duplicated task id. return } - // Get the container manifest from the taskInfo. - var manifest api.ContainerManifest - if err := yaml.Unmarshal(taskInfo.GetData(), &manifest); err != nil { + // Get the bound pod spec from the taskInfo. + var pod api.BoundPod + if err := yaml.Unmarshal(taskInfo.GetData(), &pod); err != nil { log.Warningf("Failed to extract yaml data from the taskInfo.data %v\n", err) k.sendStatusUpdate(taskInfo.GetTaskId(), mesos.TaskState_TASK_FAILED, "Failed to extract yaml data") return } - // TODO(nnielsen): Verify this assumption. Manifest ID's has been marked - // to be deprecated. - podID := manifest.ID - uuid := manifest.UUID + podFullName := kubelet.GetPodFullName(&api.BoundPod{ + ObjectMeta: api.ObjectMeta{ + Name: pod.Name, + Namespace: pod.Namespace, + Annotations: map[string]string{kubelet.ConfigSourceAnnotationKey: k.sourcename}, + }, + }) // Add the task. k.tasks[taskId] = &kuberTask{ - mesosTaskInfo: taskInfo, - containerManifest: &manifest, + mesosTaskInfo: taskInfo, + podName: podFullName, } - - pod := kubelet.Pod{ - Name: podID, - Namespace: k.namespace, - Manifest: manifest, - } - k.pods[podID] = &pod + k.pods[pod.Name] = &pod getPidInfo := func(name string) (api.PodInfo, error) { - podFullName := kubelet.GetPodFullName(&kubelet.Pod{Name: name, Namespace: k.namespace}) - return k.kl.GetPodInfo(podFullName, uuid) + return k.kl.GetPodInfo(name, "") } // TODO(nnielsen): Fail if container is already running. @@ -147,7 +143,7 @@ func (k *KubernetesExecutor) LaunchTask(driver mesos.ExecutorDriver, taskInfo *m // there is no existing event / push model for this. time.Sleep(containerPollTime) - info, err := getPidInfo(podID) + info, err := getPidInfo(podFullName) if err != nil { continue } @@ -158,7 +154,7 @@ func (k *KubernetesExecutor) LaunchTask(driver mesos.ExecutorDriver, taskInfo *m statusUpdate := &mesos.TaskStatus{ TaskId: taskInfo.GetTaskId(), State: mesos.NewTaskState(mesos.TaskState_TASK_RUNNING), - Message: proto.String("Pod '" + podID + "' is running"), + Message: proto.String("Pod '" + podFullName + "' is running"), Data: data, } @@ -171,7 +167,7 @@ func (k *KubernetesExecutor) LaunchTask(driver mesos.ExecutorDriver, taskInfo *m // What if the docker daemon is restarting and we can't connect, but it's // going to bring the pods back online as soon as it restarts? knownPod := func() bool { - _, err := getPidInfo(podID) + _, err := getPidInfo(podFullName) return err == nil } go func() { @@ -247,9 +243,7 @@ func (k *KubernetesExecutor) killPodForTask(tid, reason string) { } delete(k.tasks, tid) - // TODO(nnielsen): Verify this assumption. Manifest ID's has been marked - // to be deprecated. - pid := task.containerManifest.ID + pid := task.podName if _, found := k.pods[pid]; !found { log.Warningf("Cannot remove Unknown pod %v for task %v", pid, tid) } else { @@ -278,9 +272,7 @@ func (k *KubernetesExecutor) reportLostTask(tid, reason string) { } delete(k.tasks, tid) - // TODO(nnielsen): Verify this assumption. Manifest ID's has been marked - // to be deprecated. - pid := task.containerManifest.ID + pid := task.podName if _, found := k.pods[pid]; !found { log.Warningf("Cannot remove Unknown pod %v for lost task %v", pid, tid) } else { diff --git a/executor/server.go b/executor/server.go index 65a49c32..8a9f7477 100644 --- a/executor/server.go +++ b/executor/server.go @@ -30,6 +30,7 @@ import ( "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest" "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz" "github.com/GoogleCloudPlatform/kubernetes/pkg/httplog" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet" @@ -41,15 +42,15 @@ import ( // Server is a http.Handler which exposes kubelet functionality over HTTP. type Server struct { - host HostInterface - mux *http.ServeMux - namespace string + host HostInterface + mux *http.ServeMux + sourcename string } // ListenAndServeKubeletServer initializes a server to respond to HTTP network requests on the Kubelet. -func ListenAndServeKubeletServer(host HostInterface, address net.IP, port uint, enableDebuggingHandlers bool, namespace string) error { +func ListenAndServeKubeletServer(host HostInterface, address net.IP, port uint, enableDebuggingHandlers bool, sourcename string) error { glog.Infof("Starting to listen on %s:%d", address, port) - handler := NewServer(host, enableDebuggingHandlers, namespace) + handler := NewServer(host, enableDebuggingHandlers, sourcename) s := &http.Server{ Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)), Handler: &handler, @@ -66,6 +67,7 @@ type HostInterface interface { GetContainerInfo(podFullName, uuid, containerName string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) GetRootInfo(req *info.ContainerInfoRequest) (*info.ContainerInfo, error) GetMachineInfo() (*info.MachineInfo, error) + GetBoundPods() ([]api.BoundPod, error) GetPodInfo(name, uuid string) (api.PodInfo, error) GetKubeletContainerLogs(podFullName, containerName, tail string, follow bool, stdout, stderr io.Writer) error ServeLogs(w http.ResponseWriter, req *http.Request) @@ -74,9 +76,9 @@ type HostInterface interface { // NewServer initializes and configures a kubelet.Server object to handle HTTP requests. func NewServer(host HostInterface, enableDebuggingHandlers bool, ns string) Server { server := Server{ - host: host, - mux: http.NewServeMux(), - namespace: ns, + host: host, + mux: http.NewServeMux(), + sourcename: ns, } server.InstallDefaultHandlers() if enableDebuggingHandlers { @@ -90,6 +92,7 @@ func (s *Server) InstallDefaultHandlers() { healthz.InstallHandler(s.mux) profile.InstallHandler(s.mux) s.mux.HandleFunc("/podInfo", s.handlePodInfo) + s.mux.HandleFunc("/boundPods", s.handleBoundPods) s.mux.HandleFunc("/stats/", s.handleStats) s.mux.HandleFunc("/spec/", s.handleSpec) } @@ -115,10 +118,12 @@ func (s *Server) handleContainerLogs(w http.ResponseWriter, req *http.Request) { } parts := strings.Split(u.Path, "/") - var podID, containerName string - if len(parts) == 4 { - podID = parts[2] - containerName = parts[3] + // req URI: /containerLogs/// + var podNamespace, podID, containerName string + if len(parts) == 5 { + podNamespace = parts[2] + podID = parts[3] + containerName = parts[4] } else { http.Error(w, "Unexpected path for command running", http.StatusBadRequest) return @@ -132,16 +137,28 @@ func (s *Server) handleContainerLogs(w http.ResponseWriter, req *http.Request) { http.Error(w, `{"message": "Missing container name."}`, http.StatusBadRequest) return } + if len(podNamespace) == 0 { + http.Error(w, `{"message": "Missing podNamespace."}`, http.StatusBadRequest) + return + } uriValues := u.Query() follow, _ := strconv.ParseBool(uriValues.Get("follow")) tail := uriValues.Get("tail") - podFullName := kubelet.GetPodFullName(&kubelet.Pod{Name: podID, Namespace: s.namespace}) + podFullName := kubelet.GetPodFullName(&api.BoundPod{ + ObjectMeta: api.ObjectMeta{ + Name: podID, + Namespace: podNamespace, + Annotations: map[string]string{kubelet.ConfigSourceAnnotationKey: s.sourcename}, + }, + }) fw := FlushWriter{writer: w} if flusher, ok := w.(http.Flusher); ok { fw.flusher = flusher + } else { + s.error(w, fmt.Errorf("Unable to convert %v into http.Flusher", fw)) } w.Header().Set("Transfer-Encoding", "chunked") w.WriteHeader(http.StatusOK) @@ -152,6 +169,26 @@ func (s *Server) handleContainerLogs(w http.ResponseWriter, req *http.Request) { } } +// handleBoundPods returns a list of pod bound to the Kubelet and their spec +func (s *Server) handleBoundPods(w http.ResponseWriter, req *http.Request) { + pods, err := s.host.GetBoundPods() + if err != nil { + s.error(w, err) + return + } + boundPods := &api.BoundPods{ + Items: pods, + } + data, err := latest.Codec.Encode(boundPods) + if err != nil { + s.error(w, err) + return + } + w.WriteHeader(http.StatusOK) + w.Header().Add("Content-type", "application/json") + w.Write(data) +} + // handlePodInfo handles podInfo requests against the Kubelet. func (s *Server) handlePodInfo(w http.ResponseWriter, req *http.Request) { req.Close = true @@ -162,13 +199,25 @@ func (s *Server) handlePodInfo(w http.ResponseWriter, req *http.Request) { } podID := u.Query().Get("podID") podUUID := u.Query().Get("UUID") + podNamespace := u.Query().Get("podNamespace") if len(podID) == 0 { w.WriteHeader(http.StatusBadRequest) http.Error(w, "Missing 'podID=' query entry.", http.StatusBadRequest) return } - // TODO: backwards compatibility with existing API, needs API change - podFullName := kubelet.GetPodFullName(&kubelet.Pod{Name: podID, Namespace: s.namespace}) + if len(podNamespace) == 0 { + w.WriteHeader(http.StatusBadRequest) + http.Error(w, "Missing 'podNamespace=' query entry.", http.StatusBadRequest) + return + } + // TODO(k8s): backwards compatibility with existing API, needs API change + podFullName := kubelet.GetPodFullName(&api.BoundPod{ + ObjectMeta: api.ObjectMeta{ + Name: podID, + Namespace: podNamespace, + Annotations: map[string]string{kubelet.ConfigSourceAnnotationKey: s.sourcename}, + }, + }) info, err := s.host.GetPodInfo(podFullName, podUUID) if err == dockertools.ErrNoContainersInPod { http.Error(w, "Pod does not exist", http.StatusNotFound) @@ -248,10 +297,24 @@ func (s *Server) serveStats(w http.ResponseWriter, req *http.Request) { errors.New("pod level status currently unimplemented") case 3: // Backward compatibility without uuid information - podFullName := kubelet.GetPodFullName(&kubelet.Pod{Name: components[1], Namespace: s.namespace}) + podFullName := kubelet.GetPodFullName(&api.BoundPod{ + ObjectMeta: api.ObjectMeta{ + Name: components[1], + // TODO(k8s): I am broken + Namespace: api.NamespaceDefault, + Annotations: map[string]string{kubelet.ConfigSourceAnnotationKey: s.sourcename}, + }, + }) stats, err = s.host.GetContainerInfo(podFullName, "", components[2], &query) case 4: - podFullName := kubelet.GetPodFullName(&kubelet.Pod{Name: components[1], Namespace: s.namespace}) + podFullName := kubelet.GetPodFullName(&api.BoundPod{ + ObjectMeta: api.ObjectMeta{ + Name: components[1], + // TODO(k8s): I am broken + Namespace: "", + Annotations: map[string]string{kubelet.ConfigSourceAnnotationKey: s.sourcename}, + }, + }) stats, err = s.host.GetContainerInfo(podFullName, components[2], components[2], &query) default: http.Error(w, "unknown resource.", http.StatusNotFound) diff --git a/hack/patches/k8s---issue77.patch b/hack/patches/k8s---issue77.patch index 29439a5a..1b425245 100644 --- a/hack/patches/k8s---issue77.patch +++ b/hack/patches/k8s---issue77.patch @@ -1,8 +1,28 @@ diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go -index 2f1bb12..46464ce 100644 +index f14cb61..a8e99ed 100644 --- a/pkg/kubelet/dockertools/docker.go +++ b/pkg/kubelet/dockertools/docker.go -@@ -208,6 +208,9 @@ func GetKubeletDockerContainers(client DockerInterface, allContainers bool) (Doc +@@ -255,6 +255,9 @@ type DockerContainers map[DockerID]*docker.APIContainers + + func (c DockerContainers) FindPodContainer(podFullName, uuid, containerName string) (*docker.APIContainers, bool, uint64) { + for _, dockerContainer := range c { ++ if len(dockerContainer.Names) == 0 { ++ continue ++ } + // TODO(proppy): build the docker container name and do a map lookup instead? + dockerManifestID, dockerUUID, dockerContainerName, hash := ParseDockerName(dockerContainer.Names[0]) + if dockerManifestID == podFullName && +@@ -271,6 +274,9 @@ func (c DockerContainers) FindContainersByPodFullName(podFullName string) map[st + containers := make(map[string]*docker.APIContainers) + + for _, dockerContainer := range c { ++ if len(dockerContainer.Names) == 0 { ++ continue ++ } + dockerManifestID, _, dockerContainerName, _ := ParseDockerName(dockerContainer.Names[0]) + if dockerManifestID == podFullName { + containers[dockerContainerName] = dockerContainer +@@ -289,6 +295,9 @@ func GetKubeletDockerContainers(client DockerInterface, allContainers bool) (Doc } for i := range containers { container := &containers[i] @@ -12,7 +32,7 @@ index 2f1bb12..46464ce 100644 // Skip containers that we didn't create to allow users to manually // spin up their own containers if they want. // TODO(dchen1107): Remove the old separator "--" by end of Oct -@@ -230,6 +233,9 @@ func GetRecentDockerContainersWithNameAndUUID(client DockerInterface, podFullNam +@@ -311,6 +320,9 @@ func GetRecentDockerContainersWithNameAndUUID(client DockerInterface, podFullNam return nil, err } for _, dockerContainer := range containers { @@ -22,7 +42,7 @@ index 2f1bb12..46464ce 100644 dockerPodName, dockerUUID, dockerContainerName, _ := ParseDockerName(dockerContainer.Names[0]) if dockerPodName != podFullName { continue -@@ -340,6 +346,9 @@ func GetDockerPodInfo(client DockerInterface, manifest api.ContainerManifest, po +@@ -439,6 +451,9 @@ func GetDockerPodInfo(client DockerInterface, manifest api.PodSpec, podFullName, } for _, value := range containers { diff --git a/kubernetes-executor/main.go b/kubernetes-executor/main.go index c0927367..a824a7e7 100644 --- a/kubernetes-executor/main.go +++ b/kubernetes-executor/main.go @@ -14,11 +14,14 @@ import ( "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/record" + "github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth" "github.com/GoogleCloudPlatform/kubernetes/pkg/health" _ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet" kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config" - "github.com/GoogleCloudPlatform/kubernetes/pkg/master" + "github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag" "github.com/coreos/go-etcd/etcd" @@ -29,14 +32,14 @@ import ( ) const ( - POD_NS string = "mesos" // k8s pod namespace - defaultRootDir = "/var/lib/kubelet" + MESOS_CFG_SOURCE = "mesos" // @see ConfigSourceAnnotationKey + defaultRootDir = "/var/lib/kubelet" ) var ( syncFrequency = flag.Duration("sync_frequency", 10*time.Second, "Max period between synchronizing running containers and config") address = util.IP(net.ParseIP("0.0.0.0")) - port = flag.Uint("port", master.KubeletPort, "The port for the info server to serve on") // TODO(jdef): use kmmaster.KubeletExecutorPort + port = flag.Uint("port", ports.KubeletPort, "The port for the info server to serve on") // TODO(jdef): use kmmaster.KubeletExecutorPort hostnameOverride = flag.String("hostname_override", "", "If non-empty, will use this string as identification instead of the actual hostname.") networkContainerImage = flag.String("network_container_image", kubelet.NetworkContainerImage, "The image that network containers in each pod will use.") dockerEndpoint = flag.String("docker_endpoint", "", "If non-empty, use this for the docker endpoint to communicate with") @@ -49,11 +52,14 @@ var ( enableDebuggingHandlers = flag.Bool("enable_debugging_handlers", true, "Enables server endpoints for log collection and local running of containers and commands. Default true.") minimumGCAge = flag.Duration("minimum_container_ttl_duration", 0, "Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'") maxContainerCount = flag.Int("maximum_dead_containers_per_container", 5, "Maximum number of old instances of a container to retain per container. Each container takes up some disk space. Default: 5.") + authPath = flag.String("auth_path", "", "Path to .kubernetes_auth file, specifying how to authenticate to API server.") + apiServerList util.StringList ) func init() { flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated") flag.Var(&address, "address", "The IP address for the info and proxy servers to serve on. Default to 0.0.0.0.") + flag.Var(&apiServerList, "api_servers", "List of Kubernetes API servers to publish events to. (ip:port), comma separated.") } func getDockerEndpoint() string { @@ -84,6 +90,27 @@ func getHostname() string { return strings.TrimSpace(string(hostname)) } +func getApiserverClient() (*client.Client, error) { + authInfo, err := clientauth.LoadFromFile(*authPath) + if err != nil { + return nil, err + } + clientConfig, err := authInfo.MergeWithConfig(client.Config{}) + if err != nil { + return nil, err + } + // TODO: adapt Kube client to support LB over several servers + if len(apiServerList) > 1 { + log.Infof("Mulitple api servers specified. Picking first one") + } + clientConfig.Host = apiServerList[0] + if c, err := client.New(&clientConfig); err != nil { + return nil, err + } else { + return c, nil + } +} + func main() { flag.Parse() @@ -95,6 +122,22 @@ func main() { etcd.SetLogger(util.NewLogger("etcd ")) + // Make an API client if possible. + if len(apiServerList) < 1 { + log.Info("No api servers specified.") + } else { + if apiClient, err := getApiserverClient(); err != nil { + log.Errorf("Unable to make apiserver client: %v", err) + } else { + // Send events to APIserver if there is a client. + log.Infof("Sending events to APIserver.") + record.StartRecording(apiClient.Events(""), "kubelet") + } + } + + // Log the events locally too. + record.StartLogging(log.Infof) + capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: *allowPrivileged, }) @@ -167,6 +210,9 @@ func main() { *registryBurst, *minimumGCAge, *maxContainerCount) + + kl.BirthCry() + go func() { util.Forever(func() { err := kl.GarbageCollectContainers() @@ -198,7 +244,7 @@ func main() { health.AddHealthChecker(&health.TCPHealthChecker{}) driver := new(mesos.MesosExecutorDriver) - kubeletExecutor := executor.New(driver, kl, cfg.Channel(POD_NS), POD_NS) + kubeletExecutor := executor.New(driver, kl, cfg.Channel(MESOS_CFG_SOURCE), MESOS_CFG_SOURCE) driver.Executor = kubeletExecutor log.V(2).Infof("Initialize executor driver...") @@ -216,7 +262,7 @@ func main() { log.Infof("Starting kubelet server...") go util.Forever(func() { // TODO(nnielsen): Don't hardwire port, but use port from resource offer. - log.Error(executor.ListenAndServeKubeletServer(kl, net.IP(address), *port, *enableDebuggingHandlers, POD_NS)) + log.Error(executor.ListenAndServeKubeletServer(kl, net.IP(address), *port, *enableDebuggingHandlers, MESOS_CFG_SOURCE)) }, 0) go runProxyService() diff --git a/kubernetes-executor/plugins.go b/kubernetes-executor/plugins.go new file mode 100644 index 00000000..b61a916e --- /dev/null +++ b/kubernetes-executor/plugins.go @@ -0,0 +1,22 @@ +/* +Copyright 2014 Google Inc. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// HACK(jdef): cloned from https://github.com/GoogleCloudPlatform/kubernetes/blob/v0.5.4/cmd/kubelet/plugins.go +package main + +// This file exists to force the desired plugin implementations to be linked. +// This should probably be part of some configuration fed into the build for a +// given binary target. +import ( + _ "github.com/GoogleCloudPlatform/kubernetes/pkg/credentialprovider/gcp" +) diff --git a/kubernetes-mesos/main.go b/kubernetes-mesos/main.go index 7ffeb564..e4a7ece9 100644 --- a/kubernetes-mesos/main.go +++ b/kubernetes-mesos/main.go @@ -29,15 +29,11 @@ import ( "code.google.com/p/goprotobuf/proto" "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator/bearertoken" - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator/tokenfile" - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/handlers" "github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/master" "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/pod" "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" - "github.com/GoogleCloudPlatform/kubernetes/pkg/ui" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag" plugin "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler" @@ -50,23 +46,47 @@ import ( ) var ( - port = flag.Uint("port", 8888, "The port to listen on. Default 8888.") + // Note: the weird ""+ in below lines seems to be the only way to get gofmt to + // arrange these text blocks sensibly. Grrr. + port = flag.Int("port", 8888, ""+ + "The port to listen on. Default 8888. It is assumed that firewall rules are "+ + "set up such that this port is not reachable from outside of the cluster. It is "+ + "further assumed that port 443 on the cluster's public address is proxied to this "+ + "port.") address = util.IP(net.ParseIP("127.0.0.1")) - apiPrefix = flag.String("api_prefix", "/api", "The prefix for API requests on the server. Default '/api'") - storageVersion = flag.String("storage_version", "", "The version to store resources with. Defaults to server preferred") - minionPort = flag.Uint("minion_port", 10250, "The port at which kubelet will be listening on the minions. Default 10250.") - healthCheckMinions = flag.Bool("health_check_minions", true, "If true, health check minions and filter unhealthy ones. Default true.") - minionCacheTTL = flag.Duration("minion_cache_ttl", 30*time.Second, "Duration of time to cache minion information. Default 30 seconds.") - eventTTL = flag.Duration("event_ttl", 48*time.Hour, "Amount of time to retain events. Default 2 days.") - tokenAuthFile = flag.String("token_auth_file", "", "If set, the file that will be used to secure the API server via token authentication.") - etcdServerList util.StringList - etcdConfigFile = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers.") - corsAllowedOriginList util.StringList - allowPrivileged = flag.Bool("allow_privileged", false, "If true, allow privileged containers. Default false.") - enableLogsSupport = flag.Bool("enable_logs_support", true, "Enables server endpoint for log collection. Default true.") - mesosMaster = flag.String("mesos_master", "localhost:5050", "Location of leading Mesos master. Default localhost:5050.") - executorPath = flag.String("executor_path", "", "Location of the kubernetes executor executable") - proxyPath = flag.String("proxy_path", "", "Location of the kubernetes proxy executable") + publicAddressOverride = flag.String("public_address_override", "", ""+ + "Public serving address. Read only port will be opened on this address, "+ + "and it is assumed that port 443 at this address will be proxied/redirected "+ + "to '-address':'-port'. If blank, the address in the first listed interface "+ + "will be used.") + readOnlyPort = flag.Int("read_only_port", 7080, ""+ + "The port from which to serve read-only resources. If 0, don't serve on a "+ + "read-only address. It is assumed that firewall rules are set up such that "+ + "this port is not reachable from outside of the cluster.") + securePort = flag.Int("secure_port", 0, "The port from which to serve HTTPS with authentication and authorization. If 0, don't serve HTTPS ") + tlsCertFile = flag.String("tls_cert_file", "", "File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert).") + tlsPrivateKeyFile = flag.String("tls_private_key_file", "", "File containing x509 private key matching --tls_cert_file.") + apiPrefix = flag.String("api_prefix", "/api", "The prefix for API requests on the server. Default '/api'") + storageVersion = flag.String("storage_version", "", "The version to store resources with. Defaults to server preferred") + healthCheckMinions = flag.Bool("health_check_minions", true, "If true, health check minions and filter unhealthy ones. Default true.") + minionCacheTTL = flag.Duration("minion_cache_ttl", 30*time.Second, "Duration of time to cache minion information. Default 30 seconds.") + eventTTL = flag.Duration("event_ttl", 48*time.Hour, "Amount of time to retain events. Default 2 days.") + tokenAuthFile = flag.String("token_auth_file", "", "If set, the file that will be used to secure the API server via token authentication.") + authorizationMode = flag.String("authorization_mode", "AlwaysAllow", "Selects how to do authorization on the secure port. One of: "+strings.Join(apiserver.AuthorizationModeChoices, ",")) + authorizationPolicyFile = flag.String("authorization_policy_file", "", "File with authorization policy in csv format, used with --authorization_mode=ABAC, on the secure port.") + etcdServerList util.StringList + etcdConfigFile = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers.") + corsAllowedOriginList util.StringList + allowPrivileged = flag.Bool("allow_privileged", false, "If true, allow privileged containers. Default false.") + portalNet util.IPNet // TODO: make this a list + enableLogsSupport = flag.Bool("enable_logs_support", true, "Enables server endpoint for log collection. Default true.") + kubeletConfig = client.KubeletConfig{ + Port: 10250, + EnableHttps: false, + } + mesosMaster = flag.String("mesos_master", "localhost:5050", "Location of leading Mesos master. Default localhost:5050.") + executorPath = flag.String("executor_path", "", "Location of the kubernetes executor executable") + proxyPath = flag.String("proxy_path", "", "Location of the kubernetes proxy executable") ) const ( @@ -79,6 +99,15 @@ func init() { flag.Var(&address, "address", "The IP address on to serve on (set to 0.0.0.0 for all interfaces). Default 127.0.0.1.") flag.Var(&etcdServerList, "etcd_servers", "Servers for the etcd (http://ip:port), comma separated") flag.Var(&corsAllowedOriginList, "cors_allowed_origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.") + flag.Var(&portalNet, "portal_net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.") + client.BindKubeletClientConfigFlags(flag.CommandLine, &kubeletConfig) +} + +// TODO(k8s): Longer term we should read this from some config store, rather than a flag. +func verifyPortalFlags() { + if portalNet.IP == nil { + log.Fatal("No -portal_net specified") + } } func newEtcd(etcdConfigFile string, etcdServerList util.StringList) (helper tools.EtcdHelper, err error) { @@ -128,6 +157,7 @@ func prepareExecutorInfo() *mesos.ExecutorInfo { //TODO(jdef): provide some way (env var?) for user's to customize executor config //TODO(jdef): set -hostname_override and -address to 127.0.0.1 if `address` is 127.0.0.1 + //TODO(jdef): kubelet can publish events to the api server, we should probably tell it our IP address executorCommand := fmt.Sprintf("./%s -v=2 -hostname_override=0.0.0.0", executorCmd) if len(etcdServerList) > 0 { etcdServerArguments := strings.Join(etcdServerList, ",") @@ -160,6 +190,7 @@ func main() { defer util.FlushLogs() verflag.PrintAndExitIfRequested() + verifyPortalFlags() if (*etcdConfigFile != "" && len(etcdServerList) != 0) || (*etcdConfigFile == "" && len(etcdServerList) == 0) { log.Fatalf("specify either -etcd_servers or -etcd_config") @@ -171,10 +202,9 @@ func main() { // TODO(nnielsen): Using default pod info getter until // MesosPodInfoGetter supports network containers. - // podInfoGetter := MesosPodInfoGetter.New(mesosPodScheduler) - podInfoGetter := &client.HTTPPodInfoGetter{ - Client: http.DefaultClient, - Port: *minionPort, + kubeletClient, err := client.NewKubeletClient(&kubeletConfig) + if err != nil { + log.Fatalf("Failure to start kubelet client: %v", err) } // TODO(k8s): expose same flags as client.BindClientConfigFlags but for a server @@ -192,6 +222,13 @@ func main() { log.Fatalf("Invalid storage version or misconfigured etcd: %v", err) } + n := net.IPNet(portalNet) + + authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(*authorizationMode, *authorizationPolicyFile) + if err != nil { + log.Fatalf("Invalid Authorization Config: %v", err) + } + // Create mesos scheduler driver. executor := prepareExecutorInfo() mesosPodScheduler := kmscheduler.New(executor, kmscheduler.FCFSScheduleFunc, client, helper) @@ -203,66 +240,102 @@ func main() { }, Scheduler: mesosPodScheduler, } - m := kmmaster.New(&kmmaster.Config{ - Client: client, - Cloud: &kmscheduler.MesosCloud{mesosPodScheduler}, - EtcdHelper: helper, - HealthCheckMinions: *healthCheckMinions, - MinionCacheTTL: *minionCacheTTL, - EventTTL: *eventTTL, - PodInfoGetter: podInfoGetter, - PRFactory: func() pod.Registry { return mesosPodScheduler }, - }) - mesosPodScheduler.Init(driver, m.GetManifestFactory()) + config := &kmmaster.Config{ + Client: client, + Cloud: &kmscheduler.MesosCloud{mesosPodScheduler}, + EtcdHelper: helper, + HealthCheckMinions: *healthCheckMinions, + EventTTL: *eventTTL, + KubeletClient: kubeletClient, + PortalNet: &n, + EnableLogsSupport: *enableLogsSupport, + EnableUISupport: true, + APIPrefix: *apiPrefix, + CorsAllowedOriginList: corsAllowedOriginList, + TokenAuthFile: *tokenAuthFile, + ReadOnlyPort: *readOnlyPort, + ReadWritePort: *port, + PublicAddress: *publicAddressOverride, + Authorizer: authorizer, + PRFactory: func() pod.Registry { return mesosPodScheduler }, + } + m := kmmaster.New(config) + mesosPodScheduler.Init(driver, m.GetBoundPodFactory()) driver.Init() defer driver.Destroy() go driver.Start() - //TODO(jdef): upstream, this runs as a separate process... but not in this distro yet + //TODO(jdef): upstream, the scheduler runs as a separate process... but not in this distro yet plugin.New(mesosPodScheduler.NewPluginConfig()).Run() - log.Fatal(run(m, net.JoinHostPort(address.String(), strconv.Itoa(int(*port))))) + log.Fatal(runApiServer(config, m)) } // Run begins serving the Kubernetes API. It never returns. -func run(m *kmmaster.Master, myAddress string) error { - mux := http.NewServeMux() - apiserver.NewAPIGroup(m.API_v1beta1()).InstallREST(mux, *apiPrefix+"/v1beta1") - apiserver.NewAPIGroup(m.API_v1beta2()).InstallREST(mux, *apiPrefix+"/v1beta2") - apiserver.InstallSupport(mux) - if *enableLogsSupport { - apiserver.InstallLogsSupport(mux) +func runApiServer(config *kmmaster.Config, m *kmmaster.Master) error { + // We serve on 3 ports. See docs/reaching_the_api.md + roLocation := "" + if *readOnlyPort != 0 { + roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort)) } - ui.InstallSupport(mux) - - handler := http.Handler(mux) - - if len(corsAllowedOriginList) > 0 { - allowedOriginRegexps, err := util.CompileRegexps(corsAllowedOriginList) - if err != nil { - log.Fatalf("Invalid CORS allowed origin, --cors_allowed_origins flag was set to %v - %v", strings.Join(corsAllowedOriginList, ","), err) + secureLocation := "" + if *securePort != 0 { + secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(*securePort)) + } + rwLocation := net.JoinHostPort(address.String(), strconv.Itoa(int(*port))) + + // See the flag commentary to understand our assumptions when opening the read-only and read-write ports. + + if roLocation != "" { + // Allow 1 read-only request per second, allow up to 20 in a burst before enforcing. + rl := util.NewTokenBucketRateLimiter(1.0, 20) + readOnlyServer := &http.Server{ + Addr: roLocation, + Handler: apiserver.RecoverPanics(apiserver.ReadOnly(apiserver.RateLimit(rl, m.InsecureHandler))), + ReadTimeout: 5 * time.Minute, + WriteTimeout: 5 * time.Minute, + MaxHeaderBytes: 1 << 20, } - handler = apiserver.CORS(handler, allowedOriginRegexps, nil, nil, "true") + log.Infof("Serving read-only insecurely on %s", roLocation) + go func() { + defer util.HandleCrash() + for { + if err := readOnlyServer.ListenAndServe(); err != nil { + log.Errorf("Unable to listen for read only traffic (%v); will try again.", err) + } + time.Sleep(15 * time.Second) + } + }() } - if len(*tokenAuthFile) != 0 { - auth, err := tokenfile.New(*tokenAuthFile) - if err != nil { - log.Fatalf("Unable to load the token authentication file '%s': %v", *tokenAuthFile, err) + if secureLocation != "" { + secureServer := &http.Server{ + Addr: secureLocation, + Handler: apiserver.RecoverPanics(m.Handler), + ReadTimeout: 5 * time.Minute, + WriteTimeout: 5 * time.Minute, + MaxHeaderBytes: 1 << 20, } - userContexts := handlers.NewUserRequestContext() - handler = handlers.NewRequestAuthenticator(userContexts, bearertoken.New(auth), handlers.Unauthorized, handler) + log.Infof("Serving securely on %s", secureLocation) + go func() { + defer util.HandleCrash() + for { + if err := secureServer.ListenAndServeTLS(*tlsCertFile, *tlsPrivateKeyFile); err != nil { + log.Errorf("Unable to listen for secure (%v); will try again.", err) + } + time.Sleep(15 * time.Second) + } + }() } - handler = apiserver.RecoverPanics(handler) - s := &http.Server{ - Addr: myAddress, - Handler: handler, - ReadTimeout: httpReadTimeout, - WriteTimeout: httpWriteTimeout, + Addr: rwLocation, + Handler: apiserver.RecoverPanics(m.InsecureHandler), + ReadTimeout: 5 * time.Minute, + WriteTimeout: 5 * time.Minute, MaxHeaderBytes: 1 << 20, } + log.Infof("Serving insecurely on %s", rwLocation) return s.ListenAndServe() } diff --git a/master/handlers.go b/master/handlers.go new file mode 100644 index 00000000..90abbfc1 --- /dev/null +++ b/master/handlers.go @@ -0,0 +1,50 @@ +/* +Copyright 2014 Google Inc. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// HACK(jdef): cloned from https://github.com/GoogleCloudPlatform/kubernetes/blob/v0.5.4/pkg/master/handlers.go +package master + +import ( + "net/http" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator" + + "github.com/emicklei/go-restful" +) + +// handleWhoAmI returns the user-string which this request is authenticated as (if any). +// Useful for debugging authentication. Always returns HTTP status okay and a human +// readable (not intended as API) description of authentication state of request. +func handleWhoAmI(auth authenticator.Request) restful.RouteFunction { + return func(req *restful.Request, resp *restful.Response) { + // This is supposed to go away, so it's not worth the effort to convert to restful + w := resp.ResponseWriter + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + if auth == nil { + w.Write([]byte("NO AUTHENTICATION SUPPORT")) + return + } + userInfo, ok, err := auth.AuthenticateRequest(req.Request) + if err != nil { + w.Write([]byte("ERROR WHILE AUTHENTICATING")) + return + } + if !ok { + w.Write([]byte("NOT AUTHENTICATED")) + return + } + w.Write([]byte("AUTHENTICATED AS " + userInfo.GetName())) + return + } +} diff --git a/master/master.go b/master/master.go index 54a20858..7b899770 100644 --- a/master/master.go +++ b/master/master.go @@ -19,6 +19,15 @@ package master // HACK: copied from upstream /pkg/master/master.go, then hacked for our purposes import ( + "bytes" + _ "expvar" + "fmt" + "net" + "net/http" + "net/url" + rt "runtime" + "strconv" + "strings" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" @@ -26,6 +35,11 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta2" "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator" + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator/bearertoken" + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator/tokenfile" + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authorizer" + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/handlers" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" kmaster "github.com/GoogleCloudPlatform/kubernetes/pkg/master" @@ -40,8 +54,13 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/service" "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" + "github.com/GoogleCloudPlatform/kubernetes/pkg/ui" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" kmscheduler "github.com/mesosphere/kubernetes-mesos/scheduler" + + "github.com/emicklei/go-restful" + "github.com/emicklei/go-restful/swagger" + "github.com/golang/glog" ) const ( @@ -52,20 +71,43 @@ type PodRegistryFactory func() pod.Registry // Config is a structure used to configure a Master. type Config struct { - Client *client.Client - Cloud cloudprovider.Interface - EtcdHelper tools.EtcdHelper - HealthCheckMinions bool - MinionCacheTTL time.Duration - EventTTL time.Duration - MinionRegexp string - PodInfoGetter client.PodInfoGetter - NodeResources api.NodeResources - PRFactory PodRegistryFactory + Client *client.Client + Cloud cloudprovider.Interface + EtcdHelper tools.EtcdHelper + HealthCheckMinions bool + MinionCacheTTL time.Duration + EventTTL time.Duration + MinionRegexp string + KubeletClient client.KubeletClient + PortalNet *net.IPNet + EnableLogsSupport bool + EnableUISupport bool + APIPrefix string + CorsAllowedOriginList util.StringList + TokenAuthFile string + Authorizer authorizer.Authorizer + + // Number of masters running; all masters must be started with the + // same value for this field. (Numbers > 1 currently untested.) + MasterCount int + + // The port on PublicAddress where a read-only server will be installed. + // Defaults to 7080 if not set. + ReadOnlyPort int + // The port on PublicAddress where a read-write server will be installed. + // Defaults to 443 if not set. + ReadWritePort int + + // If empty, the first result from net.InterfaceAddrs will be used. + PublicAddress string + + // HACK(jdef): k8s-mesos implements a custom pod registry, we reference it this way + PRFactory PodRegistryFactory } // Master contains state for a Kubernetes cluster master/api server. type Master struct { + // "Inputs", Copied from Config podRegistry pod.Registry controllerRegistry controller.Registry serviceRegistry service.Registry @@ -75,7 +117,29 @@ type Master struct { eventRegistry generic.Registry storage map[string]apiserver.RESTStorage client *client.Client - manifestFactory pod.ManifestFactory + + portalNet *net.IPNet + mux apiserver.Mux + handlerContainer *restful.Container + rootWebService *restful.WebService + enableLogsSupport bool + enableUISupport bool + apiPrefix string + corsAllowedOriginList util.StringList + tokenAuthFile string + authorizer authorizer.Authorizer + masterCount int + + readOnlyServer string + readWriteServer string + masterServices *util.Runner + + // "Outputs" + Handler http.Handler + InsecureHandler http.Handler + + // HACK(jdef): need to share this reference between the binding registry and the pod registry + boundPodFactory pod.BoundPodFactory } // NewEtcdHelper returns an EtcdHelper for the provided arguments or an error if the version @@ -88,76 +152,317 @@ func NewEtcdHelper(client tools.EtcdGetSet, version string) (helper tools.EtcdHe if err != nil { return helper, err } - return tools.EtcdHelper{client, versionInterfaces.Codec, tools.RuntimeVersionAdapter{versionInterfaces.ResourceVersioner}}, nil + return tools.EtcdHelper{client, versionInterfaces.Codec, tools.RuntimeVersionAdapter{versionInterfaces.MetadataAccessor}}, nil +} + +// setDefaults fills in any fields not set that are required to have valid data. +func setDefaults(c *Config) { + if c.PortalNet == nil { + defaultNet := "10.0.0.0/24" + glog.Warningf("Portal net unspecified. Defaulting to %v.", defaultNet) + _, portalNet, err := net.ParseCIDR(defaultNet) + if err != nil { + glog.Fatalf("Unable to parse CIDR: %v", err) + } + c.PortalNet = portalNet + } + if c.MasterCount == 0 { + // Clearly, there will be at least one master. + c.MasterCount = 1 + } + if c.ReadOnlyPort == 0 { + c.ReadOnlyPort = 7080 + } + if c.ReadWritePort == 0 { + c.ReadWritePort = 443 + } + for c.PublicAddress == "" { + // Find and use the first non-loopback address. + // TODO(k8s): potentially it'd be useful to skip the docker interface if it + // somehow is first in the list. + addrs, err := net.InterfaceAddrs() + if err != nil { + glog.Fatalf("Unable to get network interfaces: error='%v'", err) + } + found := false + for i := range addrs { + ip, _, err := net.ParseCIDR(addrs[i].String()) + if err != nil { + glog.Errorf("Error parsing '%v': %v", addrs[i], err) + continue + } + if ip.IsLoopback() { + glog.Infof("'%v' (%v) is a loopback address, ignoring.", ip, addrs[i]) + continue + } + found = true + c.PublicAddress = ip.String() + glog.Infof("Will report %v as public IP address.", ip) + break + } + if !found { + glog.Errorf("Unable to find suitible network address in list: '%v'\n"+ + "Will try again in 5 seconds. Set the public address directly to avoid this wait.", addrs) + time.Sleep(5 * time.Second) + } + } } -// New returns a new instance of Master connected to the given etcd server. +// New returns a new instance of Master from the given config. +// Certain config fields will be set to a default value if unset, +// including: +// PortalNet +// MasterCount +// ReadOnlyPort +// ReadWritePort +// PublicAddress +// Certain config fields must be specified, including: +// KubeletClient +// Public fields: +// Handler -- The returned master has a field TopHandler which is an +// http.Handler which handles all the endpoints provided by the master, +// including the API, the UI, and miscelaneous debugging endpoints. All +// these are subject to authorization and authentication. +// InsecureHandler -- an http.Handler which handles all the same +// endpoints as Handler, but no authorization and authentication is done. +// Public methods: +// HandleWithAuth -- Allows caller to add an http.Handler for an endpoint +// that uses the same authentication and authorization (if any is configured) +// as the master's built-in endpoints. +// If the caller wants to add additional endpoints not using the master's +// auth, then the caller should create a handler for those endpoints, which delegates the +// any unhandled paths to "Handler". func New(c *Config) *Master { + setDefaults(c) minionRegistry := makeMinionRegistry(c) serviceRegistry := etcd.NewRegistry(c.EtcdHelper, nil) - manifestFactory := &pod.BasicManifestFactory{ + boundPodFactory := &pod.BasicBoundPodFactory{ ServiceRegistry: serviceRegistry, } - + if c.KubeletClient == nil { + glog.Fatalf("master.New() called with config.KubeletClient == nil") + } + mx := http.NewServeMux() m := &Master{ - podRegistry: c.PRFactory(), - controllerRegistry: etcd.NewRegistry(c.EtcdHelper, nil), - serviceRegistry: serviceRegistry, - endpointRegistry: etcd.NewRegistry(c.EtcdHelper, nil), - bindingRegistry: etcd.NewRegistry(c.EtcdHelper, manifestFactory), - eventRegistry: event.NewEtcdRegistry(c.EtcdHelper, uint64(c.EventTTL.Seconds())), - minionRegistry: minionRegistry, - client: c.Client, - manifestFactory: manifestFactory, - } - m.init(c.Cloud, c.PodInfoGetter) + podRegistry: c.PRFactory(), + controllerRegistry: etcd.NewRegistry(c.EtcdHelper, nil), + serviceRegistry: serviceRegistry, + endpointRegistry: etcd.NewRegistry(c.EtcdHelper, nil), + bindingRegistry: etcd.NewRegistry(c.EtcdHelper, boundPodFactory), + eventRegistry: event.NewEtcdRegistry(c.EtcdHelper, uint64(c.EventTTL.Seconds())), + minionRegistry: minionRegistry, + client: c.Client, + portalNet: c.PortalNet, + mux: mx, + handlerContainer: NewHandlerContainer(mx), + rootWebService: new(restful.WebService), + enableLogsSupport: c.EnableLogsSupport, + enableUISupport: c.EnableUISupport, + apiPrefix: c.APIPrefix, + corsAllowedOriginList: c.CorsAllowedOriginList, + tokenAuthFile: c.TokenAuthFile, + authorizer: c.Authorizer, + + masterCount: c.MasterCount, + readOnlyServer: net.JoinHostPort(c.PublicAddress, strconv.Itoa(int(c.ReadOnlyPort))), + readWriteServer: net.JoinHostPort(c.PublicAddress, strconv.Itoa(int(c.ReadWritePort))), + boundPodFactory: boundPodFactory, + } + m.masterServices = util.NewRunner(m.serviceWriterLoop, m.roServiceWriterLoop) + m.init(c) return m } +// HandleWithAuth adds an http.Handler for pattern to an http.ServeMux +// Applies the same authentication and authorization (if any is configured) +// to the request is used for the master's built-in endpoints. +func (m *Master) HandleWithAuth(pattern string, handler http.Handler) { + // TODO: Add a way for plugged-in endpoints to translate their + // URLs into attributes that an Authorizer can understand, and have + // sensible policy defaults for plugged-in endpoints. This will be different + // for generic endpoints versus REST object endpoints. + // TODO: convert to go-restful + m.mux.Handle(pattern, handler) +} + +// HandleFuncWithAuth adds an http.Handler for pattern to an http.ServeMux +// Applies the same authentication and authorization (if any is configured) +// to the request is used for the master's built-in endpoints. +func (m *Master) HandleFuncWithAuth(pattern string, handler func(http.ResponseWriter, *http.Request)) { + // TODO: convert to go-restful + m.mux.HandleFunc(pattern, handler) +} + +func NewHandlerContainer(mux *http.ServeMux) *restful.Container { + container := restful.NewContainer() + container.ServeMux = mux + container.RecoverHandler(logStackOnRecover) + return container +} + +//TODO: Unify with RecoverPanics? +func logStackOnRecover(panicReason interface{}, httpWriter http.ResponseWriter) { + var buffer bytes.Buffer + buffer.WriteString(fmt.Sprintf("recover from panic situation: - %v\r\n", panicReason)) + for i := 2; ; i += 1 { + _, file, line, ok := rt.Caller(i) + if !ok { + break + } + buffer.WriteString(fmt.Sprintf(" %s:%d\r\n", file, line)) + } + glog.Errorln(buffer.String()) +} + func makeMinionRegistry(c *Config) minion.Registry { minionRegistry := kmscheduler.NewCloudRegistry(c.Cloud) - /* TODO(jdef): kubelet-executors may not be running on all slaves, need to be sure which slaves are running such before we start doing health checks, etc... if c.HealthCheckMinions { - minionRegistry = minion.NewHealthyRegistry(minionRegistry, &http.Client{}) - } - if c.MinionCacheTTL > 0 { - cachingMinionRegistry, err := minion.NewCachingRegistry(minionRegistry, c.MinionCacheTTL) - if err != nil { - glog.Errorf("Failed to initialize caching layer, ignoring cache.") - } else { - minionRegistry = cachingMinionRegistry - } + minionRegistry = minion.NewHealthyRegistry(minionRegistry, c.KubeletClient) } */ return minionRegistry } -func (m *Master) init(cloud cloudprovider.Interface, podInfoGetter client.PodInfoGetter) { - podCache := kmaster.NewPodCache(podInfoGetter, m.podRegistry) +func (m *Master) init(c *Config) { + podCache := kmaster.NewPodCache(c.KubeletClient, m.podRegistry) go util.Forever(func() { podCache.UpdateAllContainers() }, cachePeriod) + var userContexts = handlers.NewUserRequestContext() + var authenticator authenticator.Request + if len(c.TokenAuthFile) != 0 { + tokenAuthenticator, err := tokenfile.New(c.TokenAuthFile) + if err != nil { + glog.Fatalf("Unable to load the token authentication file '%s': %v", c.TokenAuthFile, err) + } + authenticator = bearertoken.New(tokenAuthenticator) + } + + // TODO(k8s): Factor out the core API registration m.storage = map[string]apiserver.RESTStorage{ "pods": pod.NewREST(&pod.RESTConfig{ - CloudProvider: cloud, + CloudProvider: c.Cloud, PodCache: podCache, - PodInfoGetter: podInfoGetter, + PodInfoGetter: c.KubeletClient, Registry: m.podRegistry, - Minions: m.client, + Minions: m.client.Minions(), }), "replicationControllers": controller.NewREST(m.controllerRegistry, m.podRegistry), - "services": service.NewREST(m.serviceRegistry, cloud, m.minionRegistry), + "services": service.NewREST(m.serviceRegistry, c.Cloud, m.minionRegistry, m.portalNet), "endpoints": endpoint.NewREST(m.endpointRegistry), "minions": minion.NewREST(m.minionRegistry), "events": event.NewREST(m.eventRegistry), - "bindings": binding.NewREST(m.bindingRegistry), + + // TODO(k8s): should appear only in scheduler API group. + "bindings": binding.NewREST(m.bindingRegistry), + } + + apiserver.NewAPIGroupVersion(m.API_v1beta1()).InstallREST(m.handlerContainer, c.APIPrefix, "v1beta1") + apiserver.NewAPIGroupVersion(m.API_v1beta2()).InstallREST(m.handlerContainer, c.APIPrefix, "v1beta2") + + // TODO: InstallREST should register each version automatically + versionHandler := apiserver.APIVersionHandler("v1beta1", "v1beta2") + m.rootWebService.Route(m.rootWebService.GET(c.APIPrefix).To(versionHandler)) + + apiserver.InstallSupport(m.handlerContainer, m.rootWebService) + + // TODO: use go-restful + serversToValidate := m.getServersToValidate(c) + apiserver.InstallValidator(m.mux, serversToValidate) + if c.EnableLogsSupport { + apiserver.InstallLogsSupport(m.mux) + } + if c.EnableUISupport { + ui.InstallSupport(m.mux) + } + + // TODO: install runtime/pprof handler + // See github.com/emicklei/go-restful/blob/master/examples/restful-cpuprofiler-service.go + + handler := http.Handler(m.mux.(*http.ServeMux)) + + // TODO: handle CORS and auth using go-restful + // See github.com/emicklei/go-restful/blob/master/examples/restful-CORS-filter.go, and + // github.com/emicklei/go-restful/blob/master/examples/restful-basic-authentication.go + + if len(c.CorsAllowedOriginList) > 0 { + allowedOriginRegexps, err := util.CompileRegexps(c.CorsAllowedOriginList) + if err != nil { + glog.Fatalf("Invalid CORS allowed origin, --cors_allowed_origins flag was set to %v - %v", strings.Join(c.CorsAllowedOriginList, ","), err) + } + handler = apiserver.CORS(handler, allowedOriginRegexps, nil, nil, "true") } + + m.InsecureHandler = handler + + attributeGetter := apiserver.NewRequestAttributeGetter(userContexts) + handler = apiserver.WithAuthorizationCheck(handler, attributeGetter, m.authorizer) + + // Install Authenticator + if authenticator != nil { + handler = handlers.NewRequestAuthenticator(userContexts, authenticator, handlers.Unauthorized, handler) + } + // TODO: Remove temporary _whoami handler + m.rootWebService.Route(m.rootWebService.GET("/_whoami").To(handleWhoAmI(authenticator))) + + // Install root web services + m.handlerContainer.Add(m.rootWebService) + + // TODO: Make this optional? + // Enable swagger UI and discovery API + swaggerConfig := swagger.Config{ + WebServices: m.handlerContainer.RegisteredWebServices(), + // TODO: Parameterize the path? + ApiPath: "/swaggerapi/", + // TODO: Distribute UI javascript and enable the UI + //SwaggerPath: "/swaggerui/", + //SwaggerFilePath: "/srv/apiserver/swagger/dist" + } + swagger.RegisterSwaggerService(swaggerConfig, m.handlerContainer) + + m.Handler = handler + + // TODO: Attempt clean shutdown? + m.masterServices.Start() } -func (m *Master) GetManifestFactory() pod.ManifestFactory { - return m.manifestFactory +func (m *Master) getServersToValidate(c *Config) map[string]apiserver.Server { + serversToValidate := map[string]apiserver.Server{ + "controller-manager": {Addr: "127.0.0.1", Port: 10252, Path: "/healthz"}, + "scheduler": {Addr: "127.0.0.1", Port: 10251, Path: "/healthz"}, + } + for ix, machine := range c.EtcdHelper.Client.GetCluster() { + etcdUrl, err := url.Parse(machine) + if err != nil { + glog.Errorf("Failed to parse etcd url for validation: %v", err) + continue + } + var port int + var addr string + if strings.Contains(etcdUrl.Host, ":") { + var portString string + addr, portString, err = net.SplitHostPort(etcdUrl.Host) + if err != nil { + glog.Errorf("Failed to split host/port: %s (%v)", etcdUrl.Host, err) + continue + } + port, _ = strconv.Atoi(portString) + } else { + addr = etcdUrl.Host + port = 4001 + } + serversToValidate[fmt.Sprintf("etcd-%d", ix)] = apiserver.Server{Addr: addr, Port: port, Path: "/v2/keys/"} + } + nodes, err := m.minionRegistry.ListMinions(api.NewDefaultContext()) + if err != nil { + glog.Errorf("Failed to list minions: %v", err) + } + for ix, node := range nodes.Items { + serversToValidate[fmt.Sprintf("node-%d", ix)] = apiserver.Server{Addr: node.HostIP, Port: 10250, Path: "/healthz"} + } + return serversToValidate } // API_v1beta1 returns the resources and codec for API version v1beta1. @@ -177,3 +482,8 @@ func (m *Master) API_v1beta2() (map[string]apiserver.RESTStorage, runtime.Codec, } return storage, v1beta2.Codec, "/api/v1beta2", latest.SelfLinker } + +// HACK(jdef): factory method that our custom pod registry impl needs +func (m *Master) GetBoundPodFactory() pod.BoundPodFactory { + return m.boundPodFactory +} diff --git a/master/publish.go b/master/publish.go new file mode 100644 index 00000000..505cf67a --- /dev/null +++ b/master/publish.go @@ -0,0 +1,138 @@ +/* +Copyright 2014 Google Inc. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// HACK(jdef): cloned from https://github.com/GoogleCloudPlatform/kubernetes/blob/v0.5.4/pkg/master/publish.go +package master + +import ( + "fmt" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + + "github.com/golang/glog" +) + +func (m *Master) serviceWriterLoop(stop chan struct{}) { + for { + // Update service & endpoint records. + // TODO: when it becomes possible to change this stuff, + // stop polling and start watching. + // TODO: add endpoints of all replicas, not just the elected master. + if m.readWriteServer != "" { + if err := m.createMasterServiceIfNeeded("kubernetes", 443); err != nil { + glog.Errorf("Can't create rw service: %v", err) + } + if err := m.ensureEndpointsContain("kubernetes", m.readWriteServer); err != nil { + glog.Errorf("Can't create rw endpoints: %v", err) + } + } + + select { + case <-stop: + return + case <-time.After(10 * time.Second): + } + } +} + +func (m *Master) roServiceWriterLoop(stop chan struct{}) { + for { + // Update service & endpoint records. + // TODO: when it becomes possible to change this stuff, + // stop polling and start watching. + if m.readOnlyServer != "" { + if err := m.createMasterServiceIfNeeded("kubernetes-ro", 80); err != nil { + glog.Errorf("Can't create ro service: %v", err) + } + if err := m.ensureEndpointsContain("kubernetes-ro", m.readOnlyServer); err != nil { + glog.Errorf("Can't create ro endpoints: %v", err) + } + } + + select { + case <-stop: + return + case <-time.After(10 * time.Second): + } + } +} + +// createMasterServiceIfNeeded will create the specified service if it +// doesn't already exist. +func (m *Master) createMasterServiceIfNeeded(serviceName string, port int) error { + ctx := api.NewDefaultContext() + if _, err := m.serviceRegistry.GetService(ctx, serviceName); err == nil { + // The service already exists. + return nil + } + svc := &api.Service{ + ObjectMeta: api.ObjectMeta{ + Name: serviceName, + Namespace: "default", + }, + Spec: api.ServiceSpec{ + Port: port, + // We're going to add the endpoints by hand, so this selector is mainly to + // prevent identification of other pods. This selector will be useful when + // we start hosting apiserver in a pod. + Selector: map[string]string{"provider": "kubernetes", "component": "apiserver"}, + }, + } + // Kids, don't do this at home: this is a hack. There's no good way to call the business + // logic which lives in the REST object from here. + c, err := m.storage["services"].Create(ctx, svc) + if err != nil { + return err + } + resp := <-c + if _, ok := resp.Object.(*api.Service); ok { + // If all worked, we get back an *api.Service object. + return nil + } + return fmt.Errorf("Unexpected response: %#v", resp) +} + +// ensureEndpointsContain sets the endpoints for the given service. Also removes +// excess endpoints (as determined by m.masterCount). Extra endpoints could appear +// in the list if, for example, the master starts running on a different machine, +// changing IP addresses. +func (m *Master) ensureEndpointsContain(serviceName string, endpoint string) error { + ctx := api.NewDefaultContext() + e, err := m.endpointRegistry.GetEndpoints(ctx, serviceName) + if err != nil { + e = &api.Endpoints{} + // Fill in ID if it didn't exist already + e.ObjectMeta.Name = serviceName + e.ObjectMeta.Namespace = "default" + } + found := false + for i := range e.Endpoints { + if e.Endpoints[i] == endpoint { + found = true + break + } + } + if !found { + e.Endpoints = append(e.Endpoints, endpoint) + } + if len(e.Endpoints) > m.masterCount { + // We append to the end and remove from the beginning, so this should + // converge rapidly with all masters performing this operation. + e.Endpoints = e.Endpoints[len(e.Endpoints)-m.masterCount:] + } else if found { + // We didn't make any changes, no need to actually call update. + return nil + } + return m.endpointRegistry.UpdateEndpoints(ctx, e) +} diff --git a/scheduler/cloud.go b/scheduler/cloud.go index e529eea5..0aae55ac 100644 --- a/scheduler/cloud.go +++ b/scheduler/cloud.go @@ -33,6 +33,13 @@ func (c *MesosCloud) Zones() (cloud.Zones, bool) { return nil, false } +// implementation of cloud.Interface; Mesos does not provide support for multiple clusters +func (c *MesosCloud) Clusters() (cloud.Clusters, bool) { + //TODO(jdef): we could probably implement this and always return a + //single cluster- this one. + return nil, false +} + // implementation of cloud.Instances. // IPAddress returns an IP address of the specified instance. func (c *MesosCloud) IPAddress(name string) (net.IP, error) { diff --git a/scheduler/cloud_registry.go b/scheduler/cloud_registry.go index 168a0cc5..63486a87 100644 --- a/scheduler/cloud_registry.go +++ b/scheduler/cloud_registry.go @@ -22,6 +22,10 @@ func (r *CloudRegistry) CreateMinion(api.Context, *api.Minion) error { return fmt.Errorf("unsupported") } +func (r *CloudRegistry) UpdateMinion(api.Context, *api.Minion) error { + return fmt.Errorf("unsupported") +} + func (r *CloudRegistry) GetMinion(ctx api.Context, minionId string) (*api.Minion, error) { instances, ok := r.cloud.Instances() if !ok { @@ -53,10 +57,11 @@ func (r *CloudRegistry) ListMinions(ctx api.Context) (*api.MinionList, error) { for _, m := range hostnames { minions = append(minions, *(toApiMinion(ctx, m))) } - ns, _ := api.NamespaceFrom(ctx) return &api.MinionList{ - TypeMeta: api.TypeMeta{Kind: "minionList", Namespace: ns}, - Items: minions, + TypeMeta: api.TypeMeta{ + Kind: "minionList", + }, + Items: minions, }, nil } @@ -64,8 +69,10 @@ func toApiMinion(ctx api.Context, hostname string) *api.Minion { ns, _ := api.NamespaceFrom(ctx) return &api.Minion{ TypeMeta: api.TypeMeta{ - ID: hostname, - Kind: "minion", + Kind: "minion", + }, + ObjectMeta: api.ObjectMeta{ + Name: hostname, Namespace: ns, }, HostIP: hostname, diff --git a/scheduler/offers.go b/scheduler/offers.go index ccbc7142..a4944fe5 100644 --- a/scheduler/offers.go +++ b/scheduler/offers.go @@ -222,7 +222,7 @@ func (s *offerStorage) invalidateOne(offerId string) { // Walker or when the end of the offer list is reached. Expired offers are // never passed to a Walker. func (s *offerStorage) Walk(w Walker) error { - for offerId := range s.offers.Contains() { + for offerId := range s.offers.ContainedIDs() { offer, ok := s.Get(offerId) if !ok { // offer disappeared... @@ -324,7 +324,7 @@ func (s *offerStorage) notifyListeners() { log.Warningf("unexpected listener object %v", obj) } // notify if we find an acceptable offer - for id := range s.offers.Contains() { + for id := range s.offers.ContainedIDs() { var offer PerishableOffer if offer, ok = s.Get(id); !ok || offer.HasExpired() { continue diff --git a/scheduler/pod_task.go b/scheduler/pod_task.go index 1b299ced..0d55e833 100644 --- a/scheduler/pod_task.go +++ b/scheduler/pod_task.go @@ -71,7 +71,7 @@ func (t *PodTask) FillTaskInfo(offer PerishableOffer) error { return fmt.Errorf("Offer assignment must be idempotent with task %v: %v", t, offer) } t.Offer = offer - log.V(3).Infof("Recording offer(s) %v against pod %v", details.Id, t.Pod.ID) + log.V(3).Infof("Recording offer(s) %v against pod %v", details.Id, t.Pod.UID) t.TaskInfo.TaskId = &mesos.TaskID{Value: proto.String(t.ID)} t.TaskInfo.SlaveId = details.GetSlaveId() @@ -92,7 +92,7 @@ func (t *PodTask) FillTaskInfo(offer PerishableOffer) error { // Clear offer-related details from the task, should be called if/when an offer // has already been assigned to a task but for some reason is no longer valid. func (t *PodTask) ClearTaskInfo() { - log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.ID) + log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.UID) t.Offer = nil t.TaskInfo.TaskId = nil t.TaskInfo.SlaveId = nil @@ -154,7 +154,7 @@ func (t *PodTask) AcceptOffer(offer *mesos.Offer) bool { unsatisfiedPorts := len(requiredPorts) if unsatisfiedPorts > 0 { - log.V(2).Infof("Could not schedule pod %s: %d ports could not be allocated", t.Pod.ID, unsatisfiedPorts) + log.V(2).Infof("Could not schedule pod %s: %d ports could not be allocated", t.Pod.UID, unsatisfiedPorts) return false } @@ -169,7 +169,7 @@ func (t *PodTask) AcceptOffer(offer *mesos.Offer) bool { func newPodTask(pod *api.Pod, executor *mesos.ExecutorInfo) (*PodTask, error) { taskId := uuid.NewUUID().String() task := &PodTask{ - ID: taskId, // pod.JSONBase.ID, + ID: taskId, Pod: pod, TaskInfo: new(mesos.TaskInfo), Launched: false, diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index a2356600..4cddd8e6 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -111,7 +111,7 @@ type KubernetesScheduler struct { client *client.Client podQueue *cache.FIFO - manifestFactory kpod.ManifestFactory + boundPodFactory kpod.BoundPodFactory } // New create a new KubernetesScheduler @@ -157,10 +157,10 @@ func (k *KubernetesScheduler) getTask(taskId string) (*PodTask, stateType) { return nil, stateUnknown } -func (k *KubernetesScheduler) Init(d mesos.SchedulerDriver, f kpod.ManifestFactory) { +func (k *KubernetesScheduler) Init(d mesos.SchedulerDriver, f kpod.BoundPodFactory) { k.offers.Init() k.driver = d - k.manifestFactory = f + k.boundPodFactory = f } // Registered is called when the scheduler registered with the master successfully. @@ -316,7 +316,7 @@ func (k *KubernetesScheduler) fillRunningPodInfo(task *PodTask, taskStatus *meso log.Warningf("No network settings: %#v", netContainerInfo) } } else { - log.Warningf("Couldn't find network container for %s in %v", task.Pod.ID, target) + log.Warningf("Couldn't find network container for %s in %v", task.Pod.UID, target) } } else { log.Errorf("Invalid TaskStatus.Data for task '%v': %v", task.ID, err) @@ -339,7 +339,7 @@ func (k *KubernetesScheduler) handleTaskFinished(taskStatus *mesos.TaskStatus) { log.V(2).Infof( "Received finished status for running task: '%v', running/pod task queue length = %d/%d", taskStatus, len(k.runningTasks), len(k.podToTask)) - delete(k.podToTask, task.Pod.ID) + delete(k.podToTask, task.Pod.UID) k.finishedTasks.Next().Value = taskId delete(k.runningTasks, taskId) case stateFinished: @@ -356,10 +356,10 @@ func (k *KubernetesScheduler) handleTaskFailed(taskStatus *mesos.TaskStatus) { switch task, state := k.getTask(taskId); state { case statePending: delete(k.pendingTasks, taskId) - delete(k.podToTask, task.Pod.ID) + delete(k.podToTask, task.Pod.UID) case stateRunning: delete(k.runningTasks, taskId) - delete(k.podToTask, task.Pod.ID) + delete(k.podToTask, task.Pod.UID) } } @@ -370,10 +370,10 @@ func (k *KubernetesScheduler) handleTaskKilled(taskStatus *mesos.TaskStatus) { switch task, state := k.getTask(taskId); state { case statePending: delete(k.pendingTasks, taskId) - delete(k.podToTask, task.Pod.ID) + delete(k.podToTask, task.Pod.UID) case stateRunning: delete(k.runningTasks, taskId) - delete(k.podToTask, task.Pod.ID) + delete(k.podToTask, task.Pod.UID) } } @@ -384,10 +384,10 @@ func (k *KubernetesScheduler) handleTaskLost(taskStatus *mesos.TaskStatus) { switch task, state := k.getTask(taskId); state { case statePending: delete(k.pendingTasks, taskId) - delete(k.podToTask, task.Pod.ID) + delete(k.podToTask, task.Pod.UID) case stateRunning: delete(k.runningTasks, taskId) - delete(k.podToTask, task.Pod.ID) + delete(k.podToTask, task.Pod.UID) } } @@ -432,13 +432,13 @@ func (k *KubernetesScheduler) Error(driver mesos.SchedulerDriver, message string // Schedule implements the Scheduler interface of the Kubernetes. // It returns the selectedMachine's name and error (if there's any). func (k *KubernetesScheduler) Schedule(pod api.Pod, unused algorithm.MinionLister) (string, error) { - log.Infof("Try to schedule pod %v\n", pod.ID) + log.Infof("Try to schedule pod %v\n", pod.UID) k.Lock() defer k.Unlock() - if taskID, ok := k.podToTask[pod.ID]; !ok { - return "", fmt.Errorf("Pod %s cannot be resolved to a task", pod.ID) + if taskID, ok := k.podToTask[pod.UID]; !ok { + return "", fmt.Errorf("Pod %s cannot be resolved to a task", pod.UID) } else { if task, found := k.pendingTasks[taskID]; !found { return "", fmt.Errorf("Task %s is not pending, nothing to schedule", taskID) @@ -470,20 +470,20 @@ func (k *KubernetesScheduler) doSchedule(task *PodTask) (string, error) { // implementation of scheduling plugin's NextPod func; see plugin/pkg/scheduler func (k *KubernetesScheduler) yield() *api.Pod { pod := k.podQueue.Pop().(*api.Pod) - log.V(2).Infof("About to try and schedule pod %v\n", pod.ID) + log.V(2).Infof("About to try and schedule pod %v\n", pod.UID) return pod } // implementation of scheduling plugin's Error func; see plugin/pkg/scheduler func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *api.Pod, err error) { - log.Infof("Error scheduling %v: %v; retrying", pod.ID, err) + log.Infof("Error scheduling %v: %v; retrying", pod.UID, err) backoff.gc() // Retry asynchronously. // Note that this is extremely rudimentary and we need a more real error handling path. go func() { defer util.HandleCrash() - podId := pod.ID + podId := pod.UID // did we error out because if non-matching offers? if so, register an offer // listener to be notified if/when a matching offer comes in. var offersAvailable <-chan empty @@ -520,7 +520,7 @@ func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *ap // TODO(jdef) not sure that this is strictly necessary since once the pod is schedule, only the ID is // passed around in the Pod.Registry API task.Pod = pod - k.podQueue.Add(pod.ID, pod) + k.podQueue.Add(pod.UID, pod) } else { // this state shouldn't really be possible, so I'm warning if we ever see it log.Errorf("Scheduler detected pod no longer pending: %v, will not re-queue; possible offer leak", podId) @@ -614,7 +614,7 @@ func (k *KubernetesScheduler) GetPod(ctx api.Context, podId string) (*api.Pod, e func (k *KubernetesScheduler) CreatePod(ctx api.Context, pod *api.Pod) error { log.V(2).Infof("Create pod: '%v'\n", pod) // Set current status to "Waiting". - pod.CurrentState.Status = api.PodWaiting + pod.CurrentState.Status = api.PodPending pod.CurrentState.Host = "" // DesiredState.Host == "" is a signal to the scheduler that this pod needs scheduling. pod.DesiredState.Status = api.PodRunning @@ -630,12 +630,12 @@ func (k *KubernetesScheduler) CreatePod(ctx api.Context, pod *api.Pod) error { k.Lock() defer k.Unlock() - if _, ok := k.podToTask[pod.ID]; ok { - return fmt.Errorf("Pod %s already launched. Please choose a unique pod name", pod.ID) + if _, ok := k.podToTask[pod.UID]; ok { + return fmt.Errorf("Pod %s already launched. Please choose a unique pod name", pod.UID) } - k.podQueue.Add(pod.ID, pod) - k.podToTask[pod.ID] = task.ID + k.podQueue.Add(pod.UID, pod) + k.podToTask[pod.UID] = task.ID k.pendingTasks[task.ID] = task return nil @@ -692,21 +692,19 @@ func (k *KubernetesScheduler) Bind(binding *api.Binding) error { } func (k *KubernetesScheduler) prepareTaskForLaunch(machine string, task *PodTask) error { - // TODO(k8s): move this to a watch/rectification loop. - manifest, err := k.manifestFactory.MakeManifest(machine, *task.Pod) + boundPod, err := k.boundPodFactory.MakeBoundPod(machine, task.Pod) if err != nil { - log.V(2).Infof("Failed to generate an updated manifest") + log.V(2).Infof("Failed to generate an updated boundPod") return err } - // update the manifest here to pick up things like environment variables that + // update the boundPod here to pick up things like environment variables that // pod containers will use for service discovery. the kubelet-executor uses this - // manifest to instantiate the pods and this is the last update we make before + // boundPod to instantiate the pods and this is the last update we make before // firing up the pod. - task.Pod.DesiredState.Manifest = manifest - task.TaskInfo.Data, err = yaml.Marshal(&manifest) + task.TaskInfo.Data, err = yaml.Marshal(&boundPod) if err != nil { - log.V(2).Infof("Failed to marshal the updated manifest") + log.V(2).Infof("Failed to marshal the updated boundPod") return err } return nil @@ -791,7 +789,7 @@ func FCFSScheduleFunc(r OfferRegistry, slaves map[string]*Slave, task *PodTask) if task.AcceptOffer(offer) { if p.Acquire() { acceptedOffer = p - log.V(3).Infof("Pod %v accepted offer %v", task.Pod.ID, offer.Id.GetValue()) + log.V(3).Infof("Pod %v accepted offer %v", task.Pod.UID, offer.Id.GetValue()) return true, nil // stop, we found an offer } } @@ -804,10 +802,10 @@ func FCFSScheduleFunc(r OfferRegistry, slaves map[string]*Slave, task *PodTask) return acceptedOffer, nil } if err != nil { - log.V(2).Infof("failed to find a fit for pod: %v, err = %v", task.Pod.ID, err) + log.V(2).Infof("failed to find a fit for pod: %v, err = %v", task.Pod.UID, err) return nil, err } - log.V(2).Infof("failed to find a fit for pod: %v", task.Pod.ID) + log.V(2).Infof("failed to find a fit for pod: %v", task.Pod.UID) return nil, noSuitableOffersErr } diff --git a/service/endpoints_controller.go b/service/endpoints_controller.go index 7c5e5d15..684a229c 100644 --- a/service/endpoints_controller.go +++ b/service/endpoints_controller.go @@ -48,16 +48,20 @@ func NewEndpointController(client *client.Client) EndpointController { // SyncServiceEndpoints syncs service endpoints. func (e *endpointController) SyncServiceEndpoints() error { - ctx := api.NewContext() - services, err := e.client.ListServices(ctx, labels.Everything()) + services, err := e.client.Services(api.NamespaceAll).List(labels.Everything()) if err != nil { glog.Errorf("Failed to list services: %v", err) return err } var resultErr error for _, service := range services.Items { - nsCtx := api.WithNamespace(ctx, service.Namespace) - pods, err := e.client.ListPods(nsCtx, labels.Set(service.Selector).AsSelector()) + if service.Name == "kubernetes" || service.Name == "kubernetes-ro" { + // HACK(k8s): This is a temporary hack for supporting the master services + // until we actually start running apiserver in a pod. + continue + } + glog.Infof("About to update endpoints for service %v", service.Name) + pods, err := e.client.Pods(service.Namespace).List(labels.Set(service.Spec.Selector).AsSelector()) if err != nil { glog.Errorf("Error syncing service: %#v, skipping.", service) resultErr = err @@ -66,7 +70,7 @@ func (e *endpointController) SyncServiceEndpoints() error { endpoints := []string{} for _, pod := range pods.Items { // HACK(jdef): looks up a HostPort in the container, either by port-name or matching HostPort - port, err := findPort(&pod.DesiredState.Manifest, service.ContainerPort) + port, err := findPort(&pod.DesiredState.Manifest, service.Spec.ContainerPort) if err != nil { glog.Errorf("Failed to find port for service: %v, %v", service, err) continue @@ -78,13 +82,12 @@ func (e *endpointController) SyncServiceEndpoints() error { } endpoints = append(endpoints, net.JoinHostPort(pod.CurrentState.HostIP, strconv.Itoa(port))) } - currentEndpoints, err := e.client.GetEndpoints(nsCtx, service.ID) + currentEndpoints, err := e.client.Endpoints(service.Namespace).Get(service.Name) if err != nil { - // TODO(k8s) this is brittle as all get out, refactor the client libraries to return a structured error. if errors.IsNotFound(err) { currentEndpoints = &api.Endpoints{ - TypeMeta: api.TypeMeta{ - ID: service.ID, + ObjectMeta: api.ObjectMeta{ + Name: service.Name, }, } } else { @@ -98,14 +101,14 @@ func (e *endpointController) SyncServiceEndpoints() error { if len(currentEndpoints.ResourceVersion) == 0 { // No previous endpoints, create them - _, err = e.client.CreateEndpoints(nsCtx, newEndpoints) + _, err = e.client.Endpoints(service.Namespace).Create(newEndpoints) } else { // Pre-existing if endpointsEqual(currentEndpoints, endpoints) { - glog.V(2).Infof("endpoints are equal for %s, skipping update", service.ID) + glog.V(2).Infof("endpoints are equal for %s, skipping update", service.Name) continue } - _, err = e.client.UpdateEndpoints(nsCtx, newEndpoints) + _, err = e.client.Endpoints(service.Namespace).Update(newEndpoints) } if err != nil { glog.Errorf("Error updating endpoints: %#v", err) @@ -142,12 +145,40 @@ func endpointsEqual(e *api.Endpoints, endpoints []string) bool { // findPort locates the Host port for the given manifest and portName. // HACK(jdef): return the HostPort instead of the ContainerPort for generic mesos compat. func findPort(manifest *api.ContainerManifest, portName util.IntOrString) (int, error) { - if ((portName.Kind == util.IntstrString && len(portName.StrVal) == 0) || - (portName.Kind == util.IntstrInt && portName.IntVal == 0)) && - len(manifest.Containers[0].Ports) > 0 { - return manifest.Containers[0].Ports[0].HostPort, nil + firstHostPort := 0 + if len(manifest.Containers[0].Ports) > 0 { + firstHostPort = manifest.Containers[0].Ports[0].HostPort } - if portName.Kind == util.IntstrInt { + + switch portName.Kind { + case util.IntstrString: + if len(portName.StrVal) == 0 { + if firstHostPort != 0 { + return firstHostPort, nil + } + break + } + name := portName.StrVal + for _, container := range manifest.Containers { + for _, port := range container.Ports { + if port.Name == name { + return port.HostPort, nil + } + } + } + return -1, fmt.Errorf("no suitable port %s for manifest: %s", name, manifest.ID) + case util.IntstrInt: + if portName.IntVal == 0 { + if firstHostPort != 0 { + return firstHostPort, nil + } + break + } + // HACK(jdef): slightly different semantics from upstream here: + // we ensure that if the user spec'd a port in the service that + // it actually maps to a host-port declared in the pod. upstream + // doesn't check this and happily returns the port spec'd in the + // service. p := portName.IntVal for _, container := range manifest.Containers { for _, port := range container.Ports { @@ -156,15 +187,8 @@ func findPort(manifest *api.ContainerManifest, portName util.IntOrString) (int, } } } - return -1, fmt.Errorf("no suitable port for manifest: %s", manifest.ID) - } - name := portName.StrVal - for _, container := range manifest.Containers { - for _, port := range container.Ports { - if port.Name == name { - return port.HostPort, nil - } - } + return -1, fmt.Errorf("no suitable port %d for manifest: %s", p, manifest.ID) } + // should never get this far.. return -1, fmt.Errorf("no suitable port for manifest: %s", manifest.ID) } From 5e066eb502cdc516abe64a2fc59c224468e6ca65 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 8 Dec 2014 22:26:11 +0000 Subject: [PATCH 02/26] fix many problems with pod namespacing in scheduler --- scheduler/pod_task.go | 57 +++++++++++++- scheduler/scheduler.go | 169 +++++++++++++++++++++++++++-------------- 2 files changed, 164 insertions(+), 62 deletions(-) diff --git a/scheduler/pod_task.go b/scheduler/pod_task.go index 0d55e833..d8393944 100644 --- a/scheduler/pod_task.go +++ b/scheduler/pod_task.go @@ -23,6 +23,7 @@ type PodTask struct { TaskInfo *mesos.TaskInfo Launched bool Offer PerishableOffer + podKey string } func rangeResource(name string, ports []uint64) *mesos.Resource { @@ -71,7 +72,7 @@ func (t *PodTask) FillTaskInfo(offer PerishableOffer) error { return fmt.Errorf("Offer assignment must be idempotent with task %v: %v", t, offer) } t.Offer = offer - log.V(3).Infof("Recording offer(s) %v against pod %v", details.Id, t.Pod.UID) + log.V(3).Infof("Recording offer(s) %v against pod %v", details.Id, t.Pod.Name) t.TaskInfo.TaskId = &mesos.TaskID{Value: proto.String(t.ID)} t.TaskInfo.SlaveId = details.GetSlaveId() @@ -92,7 +93,7 @@ func (t *PodTask) FillTaskInfo(offer PerishableOffer) error { // Clear offer-related details from the task, should be called if/when an offer // has already been assigned to a task but for some reason is no longer valid. func (t *PodTask) ClearTaskInfo() { - log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.UID) + log.V(3).Infof("Clearing offer(s) from pod %v", t.Pod.Name) t.Offer = nil t.TaskInfo.TaskId = nil t.TaskInfo.SlaveId = nil @@ -154,7 +155,7 @@ func (t *PodTask) AcceptOffer(offer *mesos.Offer) bool { unsatisfiedPorts := len(requiredPorts) if unsatisfiedPorts > 0 { - log.V(2).Infof("Could not schedule pod %s: %d ports could not be allocated", t.Pod.UID, unsatisfiedPorts) + log.V(2).Infof("Could not schedule pod %s: %d ports could not be allocated", t.Pod.Name, unsatisfiedPorts) return false } @@ -166,15 +167,63 @@ func (t *PodTask) AcceptOffer(offer *mesos.Offer) bool { return true } -func newPodTask(pod *api.Pod, executor *mesos.ExecutorInfo) (*PodTask, error) { +func newPodTask(ctx api.Context, pod *api.Pod, executor *mesos.ExecutorInfo) (*PodTask, error) { + key, err := makePodKey(ctx, pod.Name) + if err != nil { + return nil, err + } taskId := uuid.NewUUID().String() task := &PodTask{ ID: taskId, Pod: pod, TaskInfo: new(mesos.TaskInfo), Launched: false, + podKey: key, } task.TaskInfo.Name = proto.String("PodTask") task.TaskInfo.Executor = executor return task, nil } + +/** +HACK(jdef): we're not using etcd but k8s has implemented namespace support and +we're going to try to honor that by namespacing pod keys. Hence, the following +funcs that were stolen from: + https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.5/pkg/registry/etcd/etcd.go +**/ + +const PodPath = "/pods" + +// makeListKey constructs etcd paths to resource directories enforcing namespace rules +func makeListKey(ctx api.Context, prefix string) string { + key := prefix + ns, ok := api.NamespaceFrom(ctx) + if ok && len(ns) > 0 { + key = key + "/" + ns + } + return key +} + +// makeItemKey constructs etcd paths to a resource relative to prefix enforcing namespace rules. If no namespace is on context, it errors. +func makeItemKey(ctx api.Context, prefix string, id string) (string, error) { + key := makeListKey(ctx, prefix) + ns, ok := api.NamespaceFrom(ctx) + if !ok || len(ns) == 0 { + return "", fmt.Errorf("Invalid request. Namespace parameter required.") + } + if len(id) == 0 { + return "", fmt.Errorf("Invalid request. Id parameter required.") + } + key = key + "/" + id + return key, nil +} + +// makePodListKey constructs etcd paths to pod directories enforcing namespace rules. +func makePodListKey(ctx api.Context) string { + return makeListKey(ctx, PodPath) +} + +// makePodKey constructs etcd paths to pod items enforcing namespace rules. +func makePodKey(ctx api.Context, id string) (string, error) { + return makeItemKey(ctx, PodPath, id) +} diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index 4cddd8e6..c33f8e05 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "sync" "time" @@ -316,7 +317,7 @@ func (k *KubernetesScheduler) fillRunningPodInfo(task *PodTask, taskStatus *meso log.Warningf("No network settings: %#v", netContainerInfo) } } else { - log.Warningf("Couldn't find network container for %s in %v", task.Pod.UID, target) + log.Warningf("Couldn't find network container for %s in %v", task.podKey, target) } } else { log.Errorf("Invalid TaskStatus.Data for task '%v': %v", task.ID, err) @@ -339,7 +340,7 @@ func (k *KubernetesScheduler) handleTaskFinished(taskStatus *mesos.TaskStatus) { log.V(2).Infof( "Received finished status for running task: '%v', running/pod task queue length = %d/%d", taskStatus, len(k.runningTasks), len(k.podToTask)) - delete(k.podToTask, task.Pod.UID) + delete(k.podToTask, task.podKey) k.finishedTasks.Next().Value = taskId delete(k.runningTasks, taskId) case stateFinished: @@ -356,10 +357,10 @@ func (k *KubernetesScheduler) handleTaskFailed(taskStatus *mesos.TaskStatus) { switch task, state := k.getTask(taskId); state { case statePending: delete(k.pendingTasks, taskId) - delete(k.podToTask, task.Pod.UID) + delete(k.podToTask, task.podKey) case stateRunning: delete(k.runningTasks, taskId) - delete(k.podToTask, task.Pod.UID) + delete(k.podToTask, task.podKey) } } @@ -370,10 +371,10 @@ func (k *KubernetesScheduler) handleTaskKilled(taskStatus *mesos.TaskStatus) { switch task, state := k.getTask(taskId); state { case statePending: delete(k.pendingTasks, taskId) - delete(k.podToTask, task.Pod.UID) + delete(k.podToTask, task.podKey) case stateRunning: delete(k.runningTasks, taskId) - delete(k.podToTask, task.Pod.UID) + delete(k.podToTask, task.podKey) } } @@ -384,10 +385,10 @@ func (k *KubernetesScheduler) handleTaskLost(taskStatus *mesos.TaskStatus) { switch task, state := k.getTask(taskId); state { case statePending: delete(k.pendingTasks, taskId) - delete(k.podToTask, task.Pod.UID) + delete(k.podToTask, task.podKey) case stateRunning: delete(k.runningTasks, taskId) - delete(k.podToTask, task.Pod.UID) + delete(k.podToTask, task.podKey) } } @@ -432,13 +433,26 @@ func (k *KubernetesScheduler) Error(driver mesos.SchedulerDriver, message string // Schedule implements the Scheduler interface of the Kubernetes. // It returns the selectedMachine's name and error (if there's any). func (k *KubernetesScheduler) Schedule(pod api.Pod, unused algorithm.MinionLister) (string, error) { - log.Infof("Try to schedule pod %v\n", pod.UID) + log.Infof("Try to schedule pod %v\n", pod.Name) + + // HACK(jdef): infer context from pod namespace. i wonder if this will create + // problems down the line. the pod.Registry interface accepts Context so it's a + // little strange that the scheduler interface does not. see Bind() + ctx := api.NewDefaultContext() + if len(pod.Namespace) != 0 { + ctx = api.WithNamespace(ctx, pod.Namespace) + } + // default upstream scheduler passes pod.Name as binding.PodID + podKey, err := makePodKey(ctx, pod.Name) + if err != nil { + return "", err + } k.Lock() defer k.Unlock() - if taskID, ok := k.podToTask[pod.UID]; !ok { - return "", fmt.Errorf("Pod %s cannot be resolved to a task", pod.UID) + if taskID, ok := k.podToTask[podKey]; !ok { + return "", fmt.Errorf("Pod %s cannot be resolved to a task", podKey) } else { if task, found := k.pendingTasks[taskID]; !found { return "", fmt.Errorf("Task %s is not pending, nothing to schedule", taskID) @@ -470,28 +484,40 @@ func (k *KubernetesScheduler) doSchedule(task *PodTask) (string, error) { // implementation of scheduling plugin's NextPod func; see plugin/pkg/scheduler func (k *KubernetesScheduler) yield() *api.Pod { pod := k.podQueue.Pop().(*api.Pod) - log.V(2).Infof("About to try and schedule pod %v\n", pod.UID) + log.V(2).Infof("About to try and schedule pod %v\n", pod.Name) return pod } // implementation of scheduling plugin's Error func; see plugin/pkg/scheduler func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *api.Pod, err error) { - log.Infof("Error scheduling %v: %v; retrying", pod.UID, err) + log.Infof("Error scheduling %v: %v; retrying", pod.Name, err) backoff.gc() // Retry asynchronously. // Note that this is extremely rudimentary and we need a more real error handling path. go func() { defer util.HandleCrash() - podId := pod.UID + // HACK(jdef): infer context from pod namespace. i wonder if this will create + // problems down the line. the pod.Registry interface accepts Context so it's a + // little strange that the scheduler interface does not. see Bind() + ctx := api.NewDefaultContext() + if len(pod.Namespace) != 0 { + ctx = api.WithNamespace(ctx, pod.Namespace) + } + // default upstream scheduler passes pod.Name as binding.PodID + podKey, err := makePodKey(ctx, pod.Name) + if err != nil { + log.Errorf("Failed to build pod key, will not attempt to reschedule pod %v: %v", pod.Name, err) + return + } // did we error out because if non-matching offers? if so, register an offer // listener to be notified if/when a matching offer comes in. var offersAvailable <-chan empty if err == noSuitableOffersErr { - offersAvailable = k.offers.Listen(podId, func(offer *mesos.Offer) bool { + offersAvailable = k.offers.Listen(podKey, func(offer *mesos.Offer) bool { k.RLock() defer k.RUnlock() - if taskId, ok := k.podToTask[podId]; ok { + if taskId, ok := k.podToTask[podKey]; ok { switch task, state := k.getTask(taskId); state { case statePending: return task.AcceptOffer(offer) @@ -500,13 +526,13 @@ func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *ap return false }) } - backoff.wait(podId, offersAvailable) + backoff.wait(podKey, offersAvailable) // Get the pod again; it may have changed/been scheduled already. pod = &api.Pod{} - err := k.client.Get().Path("pods").Path(podId).Do().Into(pod) + err = k.client.Get().Namespace(pod.Namespace).Path("pods").Path(pod.Name).Do().Into(pod) if err != nil { - log.Infof("Failed to get pod %v for retry: %v; abandoning", podId, err) + log.Infof("Failed to get pod %v for retry: %v; abandoning", podKey, err) return } if pod.DesiredState.Host == "" { @@ -514,19 +540,19 @@ func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *ap k.Lock() defer k.Unlock() - if taskId, exists := k.podToTask[podId]; exists { + if taskId, exists := k.podToTask[podKey]; exists { if task, ok := k.pendingTasks[taskId]; ok && !task.hasAcceptedOffer() { // "pod" now refers to a Pod instance that is not pointed to by the PodTask, so update our records // TODO(jdef) not sure that this is strictly necessary since once the pod is schedule, only the ID is // passed around in the Pod.Registry API task.Pod = pod - k.podQueue.Add(pod.UID, pod) + k.podQueue.Add(podKey, pod) } else { // this state shouldn't really be possible, so I'm warning if we ever see it - log.Errorf("Scheduler detected pod no longer pending: %v, will not re-queue; possible offer leak", podId) + log.Errorf("Scheduler detected pod no longer pending: %v, will not re-queue; possible offer leak", podKey) } } else { - log.Infof("Scheduler detected deleted pod: %v, will not re-queue", podId) + log.Infof("Scheduler detected deleted pod: %v, will not re-queue", podKey) } } }() @@ -536,7 +562,7 @@ func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *ap func (k *KubernetesScheduler) ListPodsPredicate(ctx api.Context, filter func(*api.Pod) bool) (*api.PodList, error) { k.RLock() defer k.RUnlock() - return k.listPods(filter) + return k.listPods(ctx, filter) } // ListPods obtains a list of pods that match selector. @@ -544,24 +570,29 @@ func (k *KubernetesScheduler) ListPods(ctx api.Context, selector labels.Selector log.V(2).Infof("List pods for '%v'\n", selector) k.RLock() defer k.RUnlock() - return k.listPods(func(pod *api.Pod) bool { + return k.listPods(ctx, func(pod *api.Pod) bool { return selector.Matches(labels.Set(pod.Labels)) }) } // assumes that caller has already locked around scheduler state -func (k *KubernetesScheduler) listPods(filter func(*api.Pod) bool) (*api.PodList, error) { +func (k *KubernetesScheduler) listPods(ctx api.Context, filter func(*api.Pod) bool) (*api.PodList, error) { + prefix := makePodListKey(ctx) + "/" result := []api.Pod{} for _, task := range k.runningTasks { + if !strings.HasPrefix(task.podKey, prefix) { + continue + } pod := task.Pod - if filter(pod) { result = append(result, *pod) } } for _, task := range k.pendingTasks { + if !strings.HasPrefix(task.podKey, prefix) { + continue + } pod := task.Pod - if filter(pod) { result = append(result, *pod) } @@ -580,30 +611,35 @@ func (k *KubernetesScheduler) listPods(filter func(*api.Pod) bool) (*api.PodList // Get a specific pod. It's *very* important to return a clone of the Pod that // we've saved because our caller will likely modify it. -func (k *KubernetesScheduler) GetPod(ctx api.Context, podId string) (*api.Pod, error) { - log.V(2).Infof("Get pod '%s'\n", podId) +func (k *KubernetesScheduler) GetPod(ctx api.Context, id string) (*api.Pod, error) { + log.V(2).Infof("Get pod '%s'\n", id) + + podKey, err := makePodKey(ctx, id) + if err != nil { + return nil, err + } k.RLock() defer k.RUnlock() - taskId, exists := k.podToTask[podId] + taskId, exists := k.podToTask[podKey] if !exists { - return nil, fmt.Errorf("Could not resolve pod '%s' to task id", podId) + return nil, fmt.Errorf("Could not resolve pod '%s' to task id", podKey) } switch task, state := k.getTask(taskId); state { case statePending: - log.V(5).Infof("Pending Pod '%s': %v", podId, task.Pod) + log.V(5).Infof("Pending Pod '%s': %v", podKey, task.Pod) podCopy := *task.Pod return &podCopy, nil case stateRunning: - log.V(5).Infof("Running Pod '%s': %v", podId, task.Pod) + log.V(5).Infof("Running Pod '%s': %v", podKey, task.Pod) podCopy := *task.Pod return &podCopy, nil case stateFinished: - return nil, fmt.Errorf("Pod '%s' is finished", podId) + return nil, fmt.Errorf("Pod '%s' is finished", podKey) case stateUnknown: - return nil, fmt.Errorf("Unknown Pod %v", podId) + return nil, fmt.Errorf("Unknown Pod %v", podKey) default: return nil, fmt.Errorf("Unexpected task state %v for task %v", state, taskId) } @@ -622,7 +658,7 @@ func (k *KubernetesScheduler) CreatePod(ctx api.Context, pod *api.Pod) error { // TODO(jdef) should we make a copy of the pod object instead of just assuming that the caller is // well behaved and will not change the state of the object it has given to us? - task, err := newPodTask(pod, k.executor) + task, err := newPodTask(ctx, pod, k.executor) if err != nil { return err } @@ -630,12 +666,12 @@ func (k *KubernetesScheduler) CreatePod(ctx api.Context, pod *api.Pod) error { k.Lock() defer k.Unlock() - if _, ok := k.podToTask[pod.UID]; ok { - return fmt.Errorf("Pod %s already launched. Please choose a unique pod name", pod.UID) + if _, ok := k.podToTask[task.podKey]; ok { + return fmt.Errorf("Pod %s already launched. Please choose a unique pod name", task.podKey) } - k.podQueue.Add(pod.UID, pod) - k.podToTask[pod.UID] = task.ID + k.podQueue.Add(task.podKey, pod) + k.podToTask[task.podKey] = task.ID k.pendingTasks[task.ID] = task return nil @@ -643,13 +679,26 @@ func (k *KubernetesScheduler) CreatePod(ctx api.Context, pod *api.Pod) error { // implements binding.Registry, launches the pod-associated-task in mesos func (k *KubernetesScheduler) Bind(binding *api.Binding) error { + + // HACK(jdef): infer context from binding namespace. i wonder if this will create + // problems down the line. the pod.Registry interface accepts Context so it's a + // little strange that the scheduler interface does not + ctx := api.NewDefaultContext() + if len(binding.Namespace) != 0 { + ctx = api.WithNamespace(ctx, binding.Namespace) + } + // default upstream scheduler passes pod.Name as binding.PodID + podKey, err := makePodKey(ctx, binding.PodID) + if err != nil { + return err + } + k.Lock() defer k.Unlock() - podId := binding.PodID - taskId, exists := k.podToTask[podId] + taskId, exists := k.podToTask[podKey] if !exists { - return fmt.Errorf("Could not resolve pod '%s' to task id", podId) + return fmt.Errorf("Could not resolve pod '%s' to task id", podKey) } task, exists := k.pendingTasks[taskId] @@ -661,7 +710,7 @@ func (k *KubernetesScheduler) Bind(binding *api.Binding) error { // Schedule() and now that the offer for this task was rescinded or invalidated. // ((we should never see this here)) if !task.hasAcceptedOffer() { - return fmt.Errorf("task has not accepted a valid offer, pod %v", podId) + return fmt.Errorf("task has not accepted a valid offer, pod %v", podKey) } // By this time, there is a chance that the slave is disconnected. @@ -670,10 +719,9 @@ func (k *KubernetesScheduler) Bind(binding *api.Binding) error { // already rescinded or timed out or otherwise invalidated task.Offer.Release() task.ClearTaskInfo() - return fmt.Errorf("failed prior to launchTask due to expired offer, pod %v", podId) + return fmt.Errorf("failed prior to launchTask due to expired offer, pod %v", podKey) } - var err error if err = k.prepareTaskForLaunch(binding.Host, task); err == nil { log.V(2).Infof("Launching task : %v", task) taskList := []*mesos.TaskInfo{task.TaskInfo} @@ -688,7 +736,7 @@ func (k *KubernetesScheduler) Bind(binding *api.Binding) error { } task.Offer.Release() task.ClearTaskInfo() - return fmt.Errorf("Failed to launch task for pod %s: %v", podId, err) + return fmt.Errorf("Failed to launch task for pod %s: %v", podKey, err) } func (k *KubernetesScheduler) prepareTaskForLaunch(machine string, task *PodTask) error { @@ -718,8 +766,13 @@ func (k *KubernetesScheduler) UpdatePod(ctx api.Context, pod *api.Pod) error { } // Delete an existing pod. -func (k *KubernetesScheduler) DeletePod(ctx api.Context, podId string) error { - log.V(2).Infof("Delete pod '%s'\n", podId) +func (k *KubernetesScheduler) DeletePod(ctx api.Context, id string) error { + log.V(2).Infof("Delete pod '%s'\n", id) + + podKey, err := makePodKey(ctx, id) + if err != nil { + return err + } k.Lock() defer k.Unlock() @@ -728,11 +781,11 @@ func (k *KubernetesScheduler) DeletePod(ctx api.Context, podId string) error { // it's concurrently being scheduled (somewhere between pod scheduling and // binding) - if so, then we'll end up removing it from pendingTasks which // will abort Bind()ing - k.podQueue.Delete(podId) + k.podQueue.Delete(podKey) - taskId, exists := k.podToTask[podId] + taskId, exists := k.podToTask[podKey] if !exists { - return fmt.Errorf("Could not resolve pod '%s' to task id", podId) + return fmt.Errorf("Could not resolve pod '%s' to task id", podKey) } // determine if the task has already been launched to mesos, if not then @@ -748,7 +801,7 @@ func (k *KubernetesScheduler) DeletePod(ctx api.Context, podId string) error { task.Offer.Release() task.ClearTaskInfo() } - delete(k.podToTask, podId) + delete(k.podToTask, podKey) delete(k.pendingTasks, taskId) return nil } @@ -756,7 +809,7 @@ func (k *KubernetesScheduler) DeletePod(ctx api.Context, podId string) error { case stateRunning: killTaskId = &mesos.TaskID{Value: proto.String(task.ID)} default: - return fmt.Errorf("Cannot kill pod '%s': pod not found", podId) + return fmt.Errorf("Cannot kill pod '%s': pod not found", podKey) } // signal to watchers that the related pod is going down task.Pod.DesiredState.Host = "" @@ -789,7 +842,7 @@ func FCFSScheduleFunc(r OfferRegistry, slaves map[string]*Slave, task *PodTask) if task.AcceptOffer(offer) { if p.Acquire() { acceptedOffer = p - log.V(3).Infof("Pod %v accepted offer %v", task.Pod.UID, offer.Id.GetValue()) + log.V(3).Infof("Pod %v accepted offer %v", task.podKey, offer.Id.GetValue()) return true, nil // stop, we found an offer } } @@ -802,10 +855,10 @@ func FCFSScheduleFunc(r OfferRegistry, slaves map[string]*Slave, task *PodTask) return acceptedOffer, nil } if err != nil { - log.V(2).Infof("failed to find a fit for pod: %v, err = %v", task.Pod.UID, err) + log.V(2).Infof("failed to find a fit for pod: %v, err = %v", task.podKey, err) return nil, err } - log.V(2).Infof("failed to find a fit for pod: %v", task.Pod.UID) + log.V(2).Infof("failed to find a fit for pod: %v", task.podKey) return nil, noSuitableOffersErr } From d7a7225e756cf2272f2c3a69a7570df233f3f67e Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 9 Dec 2014 14:43:39 +0000 Subject: [PATCH 03/26] avoid potential pod leak --- scheduler/scheduler.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index c33f8e05..ff7bebdc 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -529,10 +529,19 @@ func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *ap backoff.wait(podKey, offersAvailable) // Get the pod again; it may have changed/been scheduled already. - pod = &api.Pod{} - err = k.client.Get().Namespace(pod.Namespace).Path("pods").Path(pod.Name).Do().Into(pod) + //pod = &api.Pod{} + //err = k.client.Get().Namespace(pod.Namespace).Path("pods").Path(pod.Name).Do().Into(pod) + pod, err = k.client.Pods(pod.Namespace).Get(pod.Name) if err != nil { - log.Infof("Failed to get pod %v for retry: %v; abandoning", podKey, err) + log.Warningf("Failed to get pod %v for retry: %v; abandoning", podKey, err) + + // avoid potential pod leak.. + k.Lock() + defer k.Unlock() + if taskId, exists := k.podToTask[podKey]; exists { + delete(k.pendingTasks, taskId) + } + delete(k.podToTask, podKey) return } if pod.DesiredState.Host == "" { From a0b6b1bd78246210f8fc5acb3e9bae612d92f48d Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 9 Dec 2014 14:58:42 +0000 Subject: [PATCH 04/26] eliminate hardcoded container prefix --- kubernetes-executor/main.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/kubernetes-executor/main.go b/kubernetes-executor/main.go index a824a7e7..e39da151 100644 --- a/kubernetes-executor/main.go +++ b/kubernetes-executor/main.go @@ -21,6 +21,7 @@ import ( _ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet" kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config" + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" "github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag" @@ -179,21 +180,16 @@ func main() { } // TODO(???): Destroy existing k8s containers for now - we don't know how to reconcile yet. - containers, err := dockerClient.ListContainers(docker.ListContainersOptions{All: true}) - if err == nil { + if containers, err := dockertools.GetKubeletDockerContainers(dockerClient, true); err == nil { for _, container := range containers { - log.V(2).Infof("Existing container: %v", container.Names) - - for _, containerName := range container.Names { - if strings.HasPrefix(containerName, "/k8s--") { - id := container.ID - log.V(2).Infof("Removing container: %v", id) - err = dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: id, RemoveVolumes: true}) - continue - } + id := container.ID + log.V(2).Infof("Removing container: %v", id) + if err := dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: id, RemoveVolumes: true}); err != nil { + log.Warning(err) } - } + } else { + log.Warningf("Failed to list kubelet docker containers: %v", err) } // TODO(k8s): block until all sources have delivered at least one update to the channel, or break the sync loop From d94508c0d63635f15328f5e2571ec8f6d780b3fd Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 9 Dec 2014 15:55:04 +0000 Subject: [PATCH 05/26] resolve pod selflink reference errors in scheduler --- scheduler/scheduler.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index ff7bebdc..d90b0fad 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -483,7 +483,17 @@ func (k *KubernetesScheduler) doSchedule(task *PodTask) (string, error) { // implementation of scheduling plugin's NextPod func; see plugin/pkg/scheduler func (k *KubernetesScheduler) yield() *api.Pod { + // blocking... pod := k.podQueue.Pop().(*api.Pod) + + // HACK(jdef): refresh the pod data via the client, updates things like selflink that + // the upstream scheduling controller expects to have. Will not need this once we divorce + // scheduling from the apiserver (soon I hope) + pod, err := k.client.Pods(pod.Namespace).Get(pod.Name) + if err != nil { + log.Warningf("Failed to refresh pod %v, attempting to continue: %v", pod.Name, err) + } + log.V(2).Infof("About to try and schedule pod %v\n", pod.Name) return pod } @@ -529,8 +539,6 @@ func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *ap backoff.wait(podKey, offersAvailable) // Get the pod again; it may have changed/been scheduled already. - //pod = &api.Pod{} - //err = k.client.Get().Namespace(pod.Namespace).Path("pods").Path(pod.Name).Do().Into(pod) pod, err = k.client.Pods(pod.Namespace).Get(pod.Name) if err != nil { log.Warningf("Failed to get pod %v for retry: %v; abandoning", podKey, err) From 9b163715349b88a205e99e9ace20f420d7716262 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 9 Dec 2014 16:49:13 +0000 Subject: [PATCH 06/26] fixed path to kube-proxy --- kubernetes-executor/main.go | 2 +- scheduler/scheduler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes-executor/main.go b/kubernetes-executor/main.go index e39da151..63a6839d 100644 --- a/kubernetes-executor/main.go +++ b/kubernetes-executor/main.go @@ -294,7 +294,7 @@ func runProxyService() { args = append(args, "-etcd_config="+*etcdConfigFile) } //TODO(jdef): don't hardcode name of the proxy executable here - cmd := exec.Command("./proxy", args...) + cmd := exec.Command("./kube-proxy", args...) _, err := cmd.StdoutPipe() if err != nil { log.Fatal(err) diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index d90b0fad..e5b45ab1 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -486,7 +486,7 @@ func (k *KubernetesScheduler) yield() *api.Pod { // blocking... pod := k.podQueue.Pop().(*api.Pod) - // HACK(jdef): refresh the pod data via the client, updates things like selflink that + // HACK(jdef): refresh the pod data via the client, updates things like selflink that // the upstream scheduling controller expects to have. Will not need this once we divorce // scheduling from the apiserver (soon I hope) pod, err := k.client.Pods(pod.Namespace).Get(pod.Name) From 1485e0610ae5a0eba30d4c8827cdaa127d86cef2 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 9 Dec 2014 19:48:52 +0000 Subject: [PATCH 07/26] support upstream v0.5x service host variables --- examples/guestbook/php-redis/index.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/guestbook/php-redis/index.php b/examples/guestbook/php-redis/index.php index 5774b320..8acba3a5 100644 --- a/examples/guestbook/php-redis/index.php +++ b/examples/guestbook/php-redis/index.php @@ -12,7 +12,7 @@ if ($_GET['cmd'] == 'set') { $client = new Predis\Client([ 'scheme' => 'tcp', - 'host' => getenv('SERVICE_HOST'), + 'host' => getenv('REDISMASTER_SERVICE_HOST') ?: getenv('SERVICE_HOST'), 'port' => getenv('REDISMASTER_SERVICE_PORT'), ]); $client->set($_GET['key'], $_GET['value']); @@ -25,7 +25,7 @@ } $client = new Predis\Client([ 'scheme' => 'tcp', - 'host' => getenv('SERVICE_HOST'), + 'host' => getenv('REDISMASTER_SERVICE_HOST') ?: getenv('SERVICE_HOST'), 'port' => $read_port, ]); From bda0d35236a4c9f877e17dd4850d70f1f42eaf3c Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 9 Dec 2014 20:07:29 +0000 Subject: [PATCH 08/26] fix write key for pods store --- executor/executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/executor.go b/executor/executor.go index 4e22f705..410c39df 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -113,7 +113,7 @@ func (k *KubernetesExecutor) LaunchTask(driver mesos.ExecutorDriver, taskInfo *m mesosTaskInfo: taskInfo, podName: podFullName, } - k.pods[pod.Name] = &pod + k.pods[podFullName] = &pod getPidInfo := func(name string) (api.PodInfo, error) { return k.kl.GetPodInfo(name, "") From cd360f08eead6244ef1489659463f249f36b450f Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 9 Dec 2014 20:26:06 +0000 Subject: [PATCH 09/26] force container removal --- kubernetes-executor/main.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kubernetes-executor/main.go b/kubernetes-executor/main.go index 63a6839d..052d217c 100644 --- a/kubernetes-executor/main.go +++ b/kubernetes-executor/main.go @@ -181,10 +181,14 @@ func main() { // TODO(???): Destroy existing k8s containers for now - we don't know how to reconcile yet. if containers, err := dockertools.GetKubeletDockerContainers(dockerClient, true); err == nil { + opts := docker.RemoveContainerOptions{ + RemoveVolumes: true, + Force: true, + } for _, container := range containers { - id := container.ID - log.V(2).Infof("Removing container: %v", id) - if err := dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: id, RemoveVolumes: true}); err != nil { + opts.ID = container.ID + log.V(2).Infof("Removing container: %v", opts.ID) + if err := dockerClient.RemoveContainer(opts); err != nil { log.Warning(err) } } From f5c1b4b31829b43509652b849cb5f56e238363ad Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 9 Dec 2014 20:36:43 +0000 Subject: [PATCH 10/26] initial revision --- hack/patches/k8s---issue104.patch | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 hack/patches/k8s---issue104.patch diff --git a/hack/patches/k8s---issue104.patch b/hack/patches/k8s---issue104.patch new file mode 100644 index 00000000..0e4155db --- /dev/null +++ b/hack/patches/k8s---issue104.patch @@ -0,0 +1,14 @@ +diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go +index f14cb61..86796c1 100644 +--- a/pkg/kubelet/dockertools/docker.go ++++ b/pkg/kubelet/dockertools/docker.go +@@ -222,8 +222,7 @@ func (p throttledDockerPuller) Pull(image string) error { + return fmt.Errorf("pull QPS exceeded.") + } + +-func (p dockerPuller) IsImagePresent(name string) (bool, error) { +- image, _ := parseImageName(name) ++func (p dockerPuller) IsImagePresent(image string) (bool, error) { + _, err := p.client.InspectImage(image) + if err == nil { + return true, nil From eb62d984dac66656c83ab84897c02cec46e8b407 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 15 Dec 2014 14:57:18 +0000 Subject: [PATCH 11/26] rebase to k8s v0.6.2 --- Godeps/Godeps.json | 121 ++++++++++++++++--------------- hack/patches/k8s---issue77.patch | 10 --- kubernetes-mesos/main.go | 7 +- master/master.go | 19 ++--- scheduler/cloud_registry.go | 4 +- scheduler/pod_task.go | 7 +- scheduler/scheduler.go | 31 +++----- service/endpoints_controller.go | 26 +++---- 8 files changed, 103 insertions(+), 122 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1fc9cdd4..0b816b50 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -27,143 +27,143 @@ }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/api", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/client", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/constraint", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/conversion", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/health", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/httplog", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/labels", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/master", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/binding", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/controller", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/endpoint", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/etcd", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/minion", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/pod", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/service", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/scheduler", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/service", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/tools", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/util", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/version", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/volume", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/pkg/watch", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler", - "Comment": "v0.5.4", - "Rev": "a589f85a45d33cabb5562b84d6633815f8c3579b" + "Comment": "v0.6.2", + "Rev": "729fde276613eedcd99ecf5b93f095b8deb64eb4" }, { "ImportPath": "github.com/coreos/go-etcd/etcd", @@ -176,8 +176,8 @@ }, { "ImportPath": "github.com/emicklei/go-restful", - "Comment": "v1.1.2-34-gcb26ade", - "Rev": "cb26adeb9644200cb4ec7b32be31e024696e8d00" + "Comment": "v1.1.2-38-gab99062", + "Rev": "ab990627e3546d3c6c8ab45c4d887a3d66b1b6ab" }, { "ImportPath": "github.com/fsouza/go-dockerclient", @@ -209,6 +209,11 @@ "ImportPath": "github.com/spf13/pflag", "Rev": "463bdc838f2b35e9307e91d480878bda5fff7232" }, + { + "ImportPath": "golang.org/x/net/context", + "Comment": "null-214", + "Rev": "cbcac7bb8415db9b6cb4d1ebab1dc9afbd688b97" + }, { "ImportPath": "gopkg.in/v1/yaml", "Rev": "1b9791953ba4027efaeb728c7355e542a203be5e" diff --git a/hack/patches/k8s---issue77.patch b/hack/patches/k8s---issue77.patch index 1b425245..8d40a777 100644 --- a/hack/patches/k8s---issue77.patch +++ b/hack/patches/k8s---issue77.patch @@ -32,16 +32,6 @@ index f14cb61..a8e99ed 100644 // Skip containers that we didn't create to allow users to manually // spin up their own containers if they want. // TODO(dchen1107): Remove the old separator "--" by end of Oct -@@ -311,6 +320,9 @@ func GetRecentDockerContainersWithNameAndUUID(client DockerInterface, podFullNam - return nil, err - } - for _, dockerContainer := range containers { -+ if len(dockerContainer.Names) == 0 { -+ continue -+ } - dockerPodName, dockerUUID, dockerContainerName, _ := ParseDockerName(dockerContainer.Names[0]) - if dockerPodName != podFullName { - continue @@ -439,6 +451,9 @@ func GetDockerPodInfo(client DockerInterface, manifest api.PodSpec, podFullName, } diff --git a/kubernetes-mesos/main.go b/kubernetes-mesos/main.go index e4a7ece9..f28fd7c3 100644 --- a/kubernetes-mesos/main.go +++ b/kubernetes-mesos/main.go @@ -224,6 +224,11 @@ func main() { n := net.IPNet(portalNet) + authenticator, err := apiserver.NewAuthenticatorFromTokenFile(*tokenAuthFile) + if err != nil { + log.Fatalf("Invalid Authentication Config: %v", err) + } + authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(*authorizationMode, *authorizationPolicyFile) if err != nil { log.Fatalf("Invalid Authorization Config: %v", err) @@ -252,10 +257,10 @@ func main() { EnableUISupport: true, APIPrefix: *apiPrefix, CorsAllowedOriginList: corsAllowedOriginList, - TokenAuthFile: *tokenAuthFile, ReadOnlyPort: *readOnlyPort, ReadWritePort: *port, PublicAddress: *publicAddressOverride, + Authenticator: authenticator, Authorizer: authorizer, PRFactory: func() pod.Registry { return mesosPodScheduler }, } diff --git a/master/master.go b/master/master.go index 7b899770..f7f01cec 100644 --- a/master/master.go +++ b/master/master.go @@ -36,8 +36,6 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta2" "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator" - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator/bearertoken" - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authenticator/tokenfile" "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authorizer" "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/handlers" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" @@ -84,7 +82,7 @@ type Config struct { EnableUISupport bool APIPrefix string CorsAllowedOriginList util.StringList - TokenAuthFile string + Authenticator authenticator.Request Authorizer authorizer.Authorizer // Number of masters running; all masters must be started with the @@ -126,7 +124,7 @@ type Master struct { enableUISupport bool apiPrefix string corsAllowedOriginList util.StringList - tokenAuthFile string + authenticator authenticator.Request authorizer authorizer.Authorizer masterCount int @@ -260,7 +258,7 @@ func New(c *Config) *Master { enableUISupport: c.EnableUISupport, apiPrefix: c.APIPrefix, corsAllowedOriginList: c.CorsAllowedOriginList, - tokenAuthFile: c.TokenAuthFile, + authenticator: c.Authenticator, authorizer: c.Authorizer, masterCount: c.MasterCount, @@ -331,14 +329,7 @@ func (m *Master) init(c *Config) { go util.Forever(func() { podCache.UpdateAllContainers() }, cachePeriod) var userContexts = handlers.NewUserRequestContext() - var authenticator authenticator.Request - if len(c.TokenAuthFile) != 0 { - tokenAuthenticator, err := tokenfile.New(c.TokenAuthFile) - if err != nil { - glog.Fatalf("Unable to load the token authentication file '%s': %v", c.TokenAuthFile, err) - } - authenticator = bearertoken.New(tokenAuthenticator) - } + authenticator := c.Authenticator // TODO(k8s): Factor out the core API registration m.storage = map[string]apiserver.RESTStorage{ @@ -460,7 +451,7 @@ func (m *Master) getServersToValidate(c *Config) map[string]apiserver.Server { glog.Errorf("Failed to list minions: %v", err) } for ix, node := range nodes.Items { - serversToValidate[fmt.Sprintf("node-%d", ix)] = apiserver.Server{Addr: node.HostIP, Port: 10250, Path: "/healthz"} + serversToValidate[fmt.Sprintf("node-%d", ix)] = apiserver.Server{Addr: node.Status.HostIP, Port: 10250, Path: "/healthz"} } return serversToValidate } diff --git a/scheduler/cloud_registry.go b/scheduler/cloud_registry.go index 63486a87..98524cff 100644 --- a/scheduler/cloud_registry.go +++ b/scheduler/cloud_registry.go @@ -75,6 +75,8 @@ func toApiMinion(ctx api.Context, hostname string) *api.Minion { Name: hostname, Namespace: ns, }, - HostIP: hostname, + Status: api.NodeStatus{ + HostIP: hostname, + }, } } diff --git a/scheduler/pod_task.go b/scheduler/pod_task.go index d8393944..e5f74a40 100644 --- a/scheduler/pod_task.go +++ b/scheduler/pod_task.go @@ -8,7 +8,6 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" log "github.com/golang/glog" "github.com/mesos/mesos-go/mesos" - "gopkg.in/v1/yaml" ) const ( @@ -83,10 +82,6 @@ func (t *PodTask) FillTaskInfo(offer PerishableOffer) error { if ports := rangeResource("ports", t.Ports()); ports != nil { t.TaskInfo.Resources = append(t.TaskInfo.Resources, ports) } - var err error - if t.TaskInfo.Data, err = yaml.Marshal(&t.Pod.DesiredState.Manifest); err != nil { - return err - } return nil } @@ -103,7 +98,7 @@ func (t *PodTask) ClearTaskInfo() { func (t *PodTask) Ports() []uint64 { ports := make([]uint64, 0) - for _, container := range t.Pod.DesiredState.Manifest.Containers { + for _, container := range t.Pod.Spec.Containers { // strip all port==0 from this array; k8s already knows what to do with zero- // ports (it does not create 'port bindings' on the minion-host); we need to // remove the wildcards from this array since they don't consume host resources diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index c33f8e05..23343231 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -297,27 +297,24 @@ func (k *KubernetesScheduler) handleTaskRunning(taskStatus *mesos.TaskStatus) { } func (k *KubernetesScheduler) fillRunningPodInfo(task *PodTask, taskStatus *mesos.TaskStatus) { - task.Pod.CurrentState.Status = task.Pod.DesiredState.Status - task.Pod.CurrentState.Manifest = task.Pod.DesiredState.Manifest - task.Pod.CurrentState.Host = task.Pod.DesiredState.Host - + task.Pod.Status.Phase = api.PodRunning if taskStatus.Data != nil { - var target api.PodInfo - err := json.Unmarshal(taskStatus.Data, &target) + var info api.PodInfo + err := json.Unmarshal(taskStatus.Data, &info) if err == nil { - task.Pod.CurrentState.Info = target + task.Pod.Status.Info = info /// TODO(jdef) this is problematic using default Docker networking on a default /// Docker bridge -- meaning that pod IP's are not routable across the /// k8s-mesos cluster. For now, I've duplicated logic from k8s fillPodInfo - netContainerInfo, ok := target["net"] // docker.Container + netContainerInfo, ok := info["net"] // docker.Container if ok { if netContainerInfo.PodIP != "" { - task.Pod.CurrentState.PodIP = netContainerInfo.PodIP + task.Pod.Status.PodIP = netContainerInfo.PodIP } else { log.Warningf("No network settings: %#v", netContainerInfo) } } else { - log.Warningf("Couldn't find network container for %s in %v", task.podKey, target) + log.Warningf("Couldn't find network container for %s in %v", task.podKey, info) } } else { log.Errorf("Invalid TaskStatus.Data for task '%v': %v", task.ID, err) @@ -535,7 +532,7 @@ func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *ap log.Infof("Failed to get pod %v for retry: %v; abandoning", podKey, err) return } - if pod.DesiredState.Host == "" { + if pod.Status.Host == "" { // ensure that the pod hasn't been deleted while we were trying to schedule it k.Lock() defer k.Unlock() @@ -649,12 +646,8 @@ func (k *KubernetesScheduler) GetPod(ctx api.Context, id string) (*api.Pod, erro // instead the pod is queued for scheduling. func (k *KubernetesScheduler) CreatePod(ctx api.Context, pod *api.Pod) error { log.V(2).Infof("Create pod: '%v'\n", pod) - // Set current status to "Waiting". - pod.CurrentState.Status = api.PodPending - pod.CurrentState.Host = "" - // DesiredState.Host == "" is a signal to the scheduler that this pod needs scheduling. - pod.DesiredState.Status = api.PodRunning - pod.DesiredState.Host = "" + pod.Status.Phase = api.PodPending + pod.Status.Host = "" // TODO(jdef) should we make a copy of the pod object instead of just assuming that the caller is // well behaved and will not change the state of the object it has given to us? @@ -728,7 +721,7 @@ func (k *KubernetesScheduler) Bind(binding *api.Binding) error { if err = k.driver.LaunchTasks(task.Offer.Details().Id, taskList, nil); err == nil { // we *intentionally* do not record our binding to etcd since we're not using bindings // to manage pod lifecycle - task.Pod.DesiredState.Host = binding.Host + task.Pod.Status.Host = binding.Host task.Launched = true k.offers.Invalidate(offerId) return nil @@ -812,7 +805,7 @@ func (k *KubernetesScheduler) DeletePod(ctx api.Context, id string) error { return fmt.Errorf("Cannot kill pod '%s': pod not found", podKey) } // signal to watchers that the related pod is going down - task.Pod.DesiredState.Host = "" + task.Pod.Status.Host = "" return k.driver.KillTask(killTaskId) } diff --git a/service/endpoints_controller.go b/service/endpoints_controller.go index 684a229c..a1391a8e 100644 --- a/service/endpoints_controller.go +++ b/service/endpoints_controller.go @@ -70,17 +70,17 @@ func (e *endpointController) SyncServiceEndpoints() error { endpoints := []string{} for _, pod := range pods.Items { // HACK(jdef): looks up a HostPort in the container, either by port-name or matching HostPort - port, err := findPort(&pod.DesiredState.Manifest, service.Spec.ContainerPort) + port, err := findPort(&pod, service.Spec.ContainerPort) if err != nil { glog.Errorf("Failed to find port for service: %v, %v", service, err) continue } // HACK(jdef): use HostIP instead of pod.CurrentState.PodIP for generic mesos compat - if len(pod.CurrentState.HostIP) == 0 { + if len(pod.Status.HostIP) == 0 { glog.Errorf("Failed to find a host IP for pod: %v", pod) continue } - endpoints = append(endpoints, net.JoinHostPort(pod.CurrentState.HostIP, strconv.Itoa(port))) + endpoints = append(endpoints, net.JoinHostPort(pod.Status.HostIP, strconv.Itoa(port))) } currentEndpoints, err := e.client.Endpoints(service.Namespace).Get(service.Name) if err != nil { @@ -91,7 +91,7 @@ func (e *endpointController) SyncServiceEndpoints() error { }, } } else { - glog.Errorf("Error getting endpoints: %#v", err) + glog.Errorf("Error getting endpoints: %v", err) continue } } @@ -111,7 +111,7 @@ func (e *endpointController) SyncServiceEndpoints() error { _, err = e.client.Endpoints(service.Namespace).Update(newEndpoints) } if err != nil { - glog.Errorf("Error updating endpoints: %#v", err) + glog.Errorf("Error updating endpoints: %v", err) continue } } @@ -144,10 +144,10 @@ func endpointsEqual(e *api.Endpoints, endpoints []string) bool { // findPort locates the Host port for the given manifest and portName. // HACK(jdef): return the HostPort instead of the ContainerPort for generic mesos compat. -func findPort(manifest *api.ContainerManifest, portName util.IntOrString) (int, error) { +func findPort(pod *api.Pod, portName util.IntOrString) (int, error) { firstHostPort := 0 - if len(manifest.Containers[0].Ports) > 0 { - firstHostPort = manifest.Containers[0].Ports[0].HostPort + if len(pod.Spec.Containers) > 0 && len(pod.Spec.Containers[0].Ports) > 0 { + firstHostPort = pod.Spec.Containers[0].Ports[0].HostPort } switch portName.Kind { @@ -159,14 +159,14 @@ func findPort(manifest *api.ContainerManifest, portName util.IntOrString) (int, break } name := portName.StrVal - for _, container := range manifest.Containers { + for _, container := range pod.Spec.Containers { for _, port := range container.Ports { if port.Name == name { return port.HostPort, nil } } } - return -1, fmt.Errorf("no suitable port %s for manifest: %s", name, manifest.ID) + return -1, fmt.Errorf("no suitable port %s for manifest: %s", name, pod.UID) case util.IntstrInt: if portName.IntVal == 0 { if firstHostPort != 0 { @@ -180,15 +180,15 @@ func findPort(manifest *api.ContainerManifest, portName util.IntOrString) (int, // doesn't check this and happily returns the port spec'd in the // service. p := portName.IntVal - for _, container := range manifest.Containers { + for _, container := range pod.Spec.Containers { for _, port := range container.Ports { if port.HostPort == p { return p, nil } } } - return -1, fmt.Errorf("no suitable port %d for manifest: %s", p, manifest.ID) + return -1, fmt.Errorf("no suitable port %d for manifest: %s", p, pod.UID) } // should never get this far.. - return -1, fmt.Errorf("no suitable port for manifest: %s", manifest.ID) + return -1, fmt.Errorf("no suitable port for manifest: %s", pod.UID) } From 473fe822326b53712763e0429bb0dd693d8ed484 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 15 Dec 2014 16:29:40 +0000 Subject: [PATCH 12/26] experimental support for static builds --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 905faf00..82188e12 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,16 @@ WITH_MESOS_CGO_FLAGS := \ endif +FRAMEWORK_FLAGS := -v -x -tags '$(TAGS)' + +ifneq ($(STATIC),) +FRAMEWORK_FLAGS += --ldflags '-extldflags "-static"' +endif + +ifneq ($(WITH_RACE),) +FRAMEWORK_FLAGS += -race +endif + export SHELL export KUBE_GO_PACKAGE @@ -79,7 +89,7 @@ proxy: require-godep $(KUBE_GIT_VERSION_FILE) require-vendor: framework: require-godep - env $(WITH_MESOS_CGO_FLAGS) go install -v -x -tags '$(TAGS)' $${WITH_RACE:+-race} $(FRAMEWORK_CMD) + env $(WITH_MESOS_CGO_FLAGS) go install $(FRAMEWORK_FLAGS) $(FRAMEWORK_CMD) format: require-gopath go fmt $(FRAMEWORK_CMD) $(FRAMEWORK_LIB) From c309a0d4086e8a9b5b16e3890ec59015895578fe Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Mon, 15 Dec 2014 17:15:04 +0000 Subject: [PATCH 13/26] initial revision --- hack/patches/k8s---issue107.patch | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 hack/patches/k8s---issue107.patch diff --git a/hack/patches/k8s---issue107.patch b/hack/patches/k8s---issue107.patch new file mode 100644 index 00000000..cfd011e0 --- /dev/null +++ b/hack/patches/k8s---issue107.patch @@ -0,0 +1,14 @@ +diff --git a/cmd/kube-proxy/proxy.go b/cmd/kube-proxy/proxy.go +index 61fb4f8..e07a52c 100644 +--- a/cmd/kube-proxy/proxy.go ++++ b/cmd/kube-proxy/proxy.go +@@ -119,6 +119,9 @@ func main() { + } + loadBalancer := proxy.NewLoadBalancerRR() + proxier := proxy.NewProxier(loadBalancer, net.IP(bindAddress), iptables.New(exec.New(), protocol)) ++ if proxier == nil { ++ glog.Fatalf("failed to create proxier, aborting") ++ } + // Wire proxier to handle changes to services + serviceConfig.RegisterHandler(proxier) + // And wire loadBalancer to handle changes to endpoints to services From 4199f344215a3d87acc211839b1fb26300d73324 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 16 Dec 2014 23:41:33 +0000 Subject: [PATCH 14/26] updated run-through example with latest tooling names, parameters --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40ddc551..d3c8fe4b 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,18 @@ $ cd $GOPATH # If you don't have one, create directory and set GOPATH accordingl $ mkdir -p src/github.com/mesosphere/kubernetes-mesos $ git clone https://github.com/mesosphere/kubernetes-mesos.git src/github.com/mesosphere/kubernetes-mesos $ cd src/github.com/mesosphere/kubernetes-mesos && godep restore -$ go install github.com/GoogleCloudPlatform/kubernetes/cmd/{proxy,kubecfg} +$ go install github.com/GoogleCloudPlatform/kubernetes/cmd/{kube-proxy,kubecfg,kubectl} $ go install github.com/mesosphere/kubernetes-mesos/kubernetes-{mesos,executor} $ go install github.com/mesosphere/kubernetes-mesos/controller-manager ``` ### Start the framework +**NETWORKING:** Kubernetes v0.5 introduced "Services v2" which follows a Service-per-IP model. +A consequence of this is that you must identify CIDR subnet to the Kubernetes-Mesos master that may be used for allocating IP addresses for Kubernetes services (the `-portal_net` parameter). +The value provided in the example below is purely for illustration and may need to be adjusted for your network. +See the Kubernetes [release notes](https://github.com/GoogleCloudPlatform/kubernetes/releases/tag/v0.5) for additional details regarding the new Services model. + The examples that follow assume that you are running the mesos-master, etcd, and the kubernetes-mesos framework on the same host, exposed on an IP address referred to hereafter as `${servicehost}`. If you are not running in a production setting then a single etcd instance will suffice. To run etcd, see [github.com/coreos/etcd][6], or run it via docker: @@ -84,7 +89,8 @@ $ ./bin/kubernetes-mesos \ -mesos_master=${servicehost}:5050 \ -etcd_servers=http://${servicehost}:4001 \ -executor_path=$(pwd)/bin/kubernetes-executor \ - -proxy_path=$(pwd)/bin/proxy + -proxy_path=$(pwd)/bin/kube-proxy \ + -portal_net=10.10.10.0/24 ``` For simpler execution of `kubecfg`: From b48600eea6d9874628d90f456a418c7a1a7cdf40 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Tue, 16 Dec 2014 23:50:46 +0000 Subject: [PATCH 15/26] wraped lines for readability --- hack/dockerbuild/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/dockerbuild/README.md b/hack/dockerbuild/README.md index 04031cd1..77f3e1b9 100644 --- a/hack/dockerbuild/README.md +++ b/hack/dockerbuild/README.md @@ -52,11 +52,13 @@ Want a quick-and-dirty development environment to start hacking? Need to build the project, but from a forked git repo? - $ docker run --rm -v /tmp/target:/target -e GIT_REPO=https://github.com/whoami/kubernetes-mesos jdef/kubernetes-mesos:build-latest + $ docker run --rm -v /tmp/target:/target -e GIT_REPO=https://github.com/whoami/kubernetes-mesos \ + jdef/kubernetes-mesos:build-latest To hack in your currently checked out repo mount the root of the github repo to `/snapshot`: - $ docker run -ti -v /tmp/target:/target -v /home/jdef/kubernetes-mesos:/snapshot jdef/kubernetes-mesos:build-latest bash + $ docker run -ti -v /tmp/target:/target -v /home/jdef/kubernetes-mesos:/snapshot \ + jdef/kubernetes-mesos:build-latest bash ## Profiling From bc3021f337b3db68f606ec29a94bee0de33fd2ca Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Wed, 17 Dec 2014 02:59:53 +0000 Subject: [PATCH 16/26] doc cleanup --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d3c8fe4b..37e200bb 100644 --- a/README.md +++ b/README.md @@ -62,10 +62,10 @@ $ go install github.com/mesosphere/kubernetes-mesos/controller-manager ### Start the framework -**NETWORKING:** Kubernetes v0.5 introduced "Services v2" which follows a Service-per-IP model. -A consequence of this is that you must identify CIDR subnet to the Kubernetes-Mesos master that may be used for allocating IP addresses for Kubernetes services (the `-portal_net` parameter). -The value provided in the example below is purely for illustration and may need to be adjusted for your network. -See the Kubernetes [release notes](https://github.com/GoogleCloudPlatform/kubernetes/releases/tag/v0.5) for additional details regarding the new Services model. +**NETWORKING:** Kubernetes v0.5 introduced "Services v2" which follows an IP-per-Service model. +A consequence of this is that you must provide the Kubernetes-Mesos framework with a [CIDR][8] subnet that will be used for the allocation of IP addresses for Kubernetes services: the `-portal_net` parameter. +Please keep this in mind when reviewing (and attempting) the example below - the CIDR subnet may need to be adjusted for your network. +See the Kubernetes [release notes][9] for additional details regarding the new services model. The examples that follow assume that you are running the mesos-master, etcd, and the kubernetes-mesos framework on the same host, exposed on an IP address referred to hereafter as `${servicehost}`. If you are not running in a production setting then a single etcd instance will suffice. @@ -378,3 +378,5 @@ $ go test github.com/mesosphere/kubernetes-mesos/kubernetes-mesos -v [5]: https://github.com/tools/godep [6]: https://github.com/coreos/etcd/releases/ [7]: DEVELOP.md#prerequisites +[8]: http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing +[9]: https://github.com/GoogleCloudPlatform/kubernetes/releases/tag/v0.5 From 808246b7dc23fa0db190bd8c48c8bd2d2cfc2258 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Wed, 17 Dec 2014 19:21:44 +0000 Subject: [PATCH 17/26] do not communicate TASK_RUNNING status prematurely ("No network settings" error) --- executor/executor.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/executor/executor.go b/executor/executor.go index 410c39df..73b37917 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -148,6 +148,11 @@ func (k *KubernetesExecutor) LaunchTask(driver mesos.ExecutorDriver, taskInfo *m continue } + // avoid sending back a running status while pod networking is down + if podnet, ok := info["net"]; !ok || podnet.State.Running == nil { + continue + } + log.V(2).Infof("Found pod info: '%v'", info) data, err := json.Marshal(info) From d9f34764f828a7427b5710805843852604a2f6ae Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Thu, 18 Dec 2014 13:29:22 +0000 Subject: [PATCH 18/26] initial revision --- hack/patches/k8s---issue110.patch | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 hack/patches/k8s---issue110.patch diff --git a/hack/patches/k8s---issue110.patch b/hack/patches/k8s---issue110.patch new file mode 100644 index 00000000..6093608c --- /dev/null +++ b/hack/patches/k8s---issue110.patch @@ -0,0 +1,13 @@ +diff --git a/pkg/registry/pod/rest.go b/pkg/registry/pod/rest.go +index c3c14b6..8138bae 100644 +--- a/pkg/registry/pod/rest.go ++++ b/pkg/registry/pod/rest.go +@@ -234,7 +234,7 @@ func (rs *REST) fillPodInfo(pod *api.Pod) { + if ok { + if netContainerInfo.PodIP != "" { + pod.Status.PodIP = netContainerInfo.PodIP +- } else { ++ } else if netContainerInfo.State.Running != nil { + glog.Warningf("No network settings: %#v", netContainerInfo) + } + } else { From 815de0e06dabbfee641f76ffcfc76714a41e1753 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 19 Dec 2014 05:04:44 +0000 Subject: [PATCH 19/26] only use updated pod spec if query was successful, otherwise use cached version - avoids using incomplete pod spec on refresh errors --- scheduler/scheduler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index 14b2e5f9..d79a8430 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -486,9 +486,11 @@ func (k *KubernetesScheduler) yield() *api.Pod { // HACK(jdef): refresh the pod data via the client, updates things like selflink that // the upstream scheduling controller expects to have. Will not need this once we divorce // scheduling from the apiserver (soon I hope) - pod, err := k.client.Pods(pod.Namespace).Get(pod.Name) + updatedPod, err := k.client.Pods(pod.Namespace).Get(pod.Name) if err != nil { log.Warningf("Failed to refresh pod %v, attempting to continue: %v", pod.Name, err) + } else { + pod = updatedPod } log.V(2).Infof("About to try and schedule pod %v\n", pod.Name) From c3b125bdb07fbe17cfb17377d0837c5f887c2b2d Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 19 Dec 2014 05:05:00 +0000 Subject: [PATCH 20/26] go fmt --- kubernetes-mesos/main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kubernetes-mesos/main.go b/kubernetes-mesos/main.go index 717c75ef..9e8443de 100644 --- a/kubernetes-mesos/main.go +++ b/kubernetes-mesos/main.go @@ -86,13 +86,13 @@ var ( Port: 10250, EnableHttps: false, } - mesosMaster = flag.String("mesos_master", "localhost:5050", "Location of leading Mesos master. Default localhost:5050.") - executorPath = flag.String("executor_path", "", "Location of the kubernetes executor executable") - proxyPath = flag.String("proxy_path", "", "Location of the kubernetes proxy executable") - mesosUser = flag.String("mesos_user", "", "Mesos user for this framework, defaults to the username that owns the framework process.") - mesosRole = flag.String("mesos_role", "", "Mesos role for this framework, defaults to none.") - mesosAuthPrincipal = flag.String("mesos_authentication_principal", "", "Mesos authentication principal.") - mesosAuthSecretFile = flag.String("mesos_authentication_secret_file", "", "Mesos authentication secret file.") + mesosMaster = flag.String("mesos_master", "localhost:5050", "Location of leading Mesos master. Default localhost:5050.") + executorPath = flag.String("executor_path", "", "Location of the kubernetes executor executable") + proxyPath = flag.String("proxy_path", "", "Location of the kubernetes proxy executable") + mesosUser = flag.String("mesos_user", "", "Mesos user for this framework, defaults to the username that owns the framework process.") + mesosRole = flag.String("mesos_role", "", "Mesos role for this framework, defaults to none.") + mesosAuthPrincipal = flag.String("mesos_authentication_principal", "", "Mesos authentication principal.") + mesosAuthSecretFile = flag.String("mesos_authentication_secret_file", "", "Mesos authentication secret file.") ) const ( From 4030b653484f8cbf0583e9f59a519d7a49110050 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Sat, 20 Dec 2014 03:05:05 +0000 Subject: [PATCH 21/26] avoid requeueing a pod for scheduling if is has already been deleted --- scheduler/scheduler.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index d79a8430..bae78423 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -45,6 +45,14 @@ var ( noSuitableOffersErr = errors.New("No suitable offers for pod/task") ) +type NoSuchTaskForPod struct { + podKey string +} + +func (err *NoSuchTaskForPod) Error() string { + return fmt.Sprintf("Pod %s cannot be resolved to a task", err.podKey) +} + // PodScheduleFunc implements how to schedule pods among slaves. // We can have different implementation for different scheduling policy. // @@ -449,7 +457,7 @@ func (k *KubernetesScheduler) Schedule(pod api.Pod, unused algorithm.MinionListe defer k.Unlock() if taskID, ok := k.podToTask[podKey]; !ok { - return "", fmt.Errorf("Pod %s cannot be resolved to a task", podKey) + return "", &NoSuchTaskForPod{podKey} } else { if task, found := k.pendingTasks[taskID]; !found { return "", fmt.Errorf("Task %s is not pending, nothing to schedule", taskID) @@ -499,6 +507,11 @@ func (k *KubernetesScheduler) yield() *api.Pod { // implementation of scheduling plugin's Error func; see plugin/pkg/scheduler func (k *KubernetesScheduler) handleSchedulingError(backoff *podBackoff, pod *api.Pod, err error) { + + if nst, ok := err.(*NoSuchTaskForPod); ok { + log.V(2).Infof("Not rescheduling pod %v since it has no mapped task", nst.podKey) + return + } log.Infof("Error scheduling %v: %v; retrying", pod.Name, err) backoff.gc() @@ -640,7 +653,7 @@ func (k *KubernetesScheduler) GetPod(ctx api.Context, id string) (*api.Pod, erro taskId, exists := k.podToTask[podKey] if !exists { - return nil, fmt.Errorf("Could not resolve pod '%s' to task id", podKey) + return nil, &NoSuchTaskForPod{podKey} } switch task, state := k.getTask(taskId); state { @@ -797,7 +810,7 @@ func (k *KubernetesScheduler) DeletePod(ctx api.Context, id string) error { taskId, exists := k.podToTask[podKey] if !exists { - return fmt.Errorf("Could not resolve pod '%s' to task id", podKey) + return &NoSuchTaskForPod{podKey} } // determine if the task has already been launched to mesos, if not then From df59a26544224225781378aa5dc5edaec2b01ab3 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 Dec 2014 04:51:00 +0000 Subject: [PATCH 22/26] executor and proxy should be marked as executable --- kubernetes-mesos/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes-mesos/main.go b/kubernetes-mesos/main.go index 9e8443de..6dbf7f78 100644 --- a/kubernetes-mesos/main.go +++ b/kubernetes-mesos/main.go @@ -157,9 +157,9 @@ func serveExecutorArtifact(path string) (*string, string) { func prepareExecutorInfo() *mesos.ExecutorInfo { executorUris := []*mesos.CommandInfo_URI{} uri, _ := serveExecutorArtifact(*proxyPath) - executorUris = append(executorUris, &mesos.CommandInfo_URI{Value: uri}) + executorUris = append(executorUris, &mesos.CommandInfo_URI{Value: uri, Executable: proto.Bool(true)}) uri, executorCmd := serveExecutorArtifact(*executorPath) - executorUris = append(executorUris, &mesos.CommandInfo_URI{Value: uri}) + executorUris = append(executorUris, &mesos.CommandInfo_URI{Value: uri, Executable: proto.Bool(true)}) //TODO(jdef): provide some way (env var?) for user's to customize executor config //TODO(jdef): set -hostname_override and -address to 127.0.0.1 if `address` is 127.0.0.1 From a352787e2ad1819dafdbad4475d3257ee1b90fd1 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 Dec 2014 05:01:03 +0000 Subject: [PATCH 23/26] set user=root for executor command info so that it can exec docker commands --- kubernetes-mesos/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes-mesos/main.go b/kubernetes-mesos/main.go index 6dbf7f78..0236eb71 100644 --- a/kubernetes-mesos/main.go +++ b/kubernetes-mesos/main.go @@ -178,11 +178,13 @@ func prepareExecutorInfo() *mesos.ExecutorInfo { log.V(2).Info("Serving executor artifacts...") // Create mesos scheduler driver. + // TODO(jdef): don't hardcode executor user here return &mesos.ExecutorInfo{ ExecutorId: &mesos.ExecutorID{Value: proto.String("KubeleteExecutorID")}, Command: &mesos.CommandInfo{ Value: proto.String(executorCommand), Uris: executorUris, + User: proto.String("root"), // must be able to use docker socket }, Name: proto.String("Kubelet Executor"), Source: proto.String("kubernetes"), From 634d33716288c430e332335808d5462af3563b9c Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 Dec 2014 05:07:22 +0000 Subject: [PATCH 24/26] Revert "set user=root for executor command info so that it can exec docker commands" This reverts commit a352787e2ad1819dafdbad4475d3257ee1b90fd1. --- kubernetes-mesos/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/kubernetes-mesos/main.go b/kubernetes-mesos/main.go index 0236eb71..6dbf7f78 100644 --- a/kubernetes-mesos/main.go +++ b/kubernetes-mesos/main.go @@ -178,13 +178,11 @@ func prepareExecutorInfo() *mesos.ExecutorInfo { log.V(2).Info("Serving executor artifacts...") // Create mesos scheduler driver. - // TODO(jdef): don't hardcode executor user here return &mesos.ExecutorInfo{ ExecutorId: &mesos.ExecutorID{Value: proto.String("KubeleteExecutorID")}, Command: &mesos.CommandInfo{ Value: proto.String(executorCommand), Uris: executorUris, - User: proto.String("root"), // must be able to use docker socket }, Name: proto.String("Kubelet Executor"), Source: proto.String("kubernetes"), From 614bf38e29348259f336b9200ce49c38c035b838 Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 Dec 2014 21:10:01 +0000 Subject: [PATCH 25/26] resolve #111 --- kubernetes-mesos/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes-mesos/main.go b/kubernetes-mesos/main.go index 6dbf7f78..b955e8a0 100644 --- a/kubernetes-mesos/main.go +++ b/kubernetes-mesos/main.go @@ -164,7 +164,7 @@ func prepareExecutorInfo() *mesos.ExecutorInfo { //TODO(jdef): provide some way (env var?) for user's to customize executor config //TODO(jdef): set -hostname_override and -address to 127.0.0.1 if `address` is 127.0.0.1 //TODO(jdef): kubelet can publish events to the api server, we should probably tell it our IP address - executorCommand := fmt.Sprintf("./%s -v=2 -hostname_override=0.0.0.0", executorCmd) + executorCommand := fmt.Sprintf("./%s -v=2 -hostname_override=0.0.0.0 -allow_privileged=%t", executorCmd, *allowPrivileged) if len(etcdServerList) > 0 { etcdServerArguments := strings.Join(etcdServerList, ",") executorCommand = fmt.Sprintf("%s -etcd_servers=%s", executorCommand, etcdServerArguments) From ca6b323564f31964f20e49e44e2c29c44987d0fd Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 26 Dec 2014 21:14:05 +0000 Subject: [PATCH 26/26] added -mesos_user=root to allow executor to access docker socket --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 37e200bb..1f82837b 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,8 @@ $ ./bin/kubernetes-mesos \ -etcd_servers=http://${servicehost}:4001 \ -executor_path=$(pwd)/bin/kubernetes-executor \ -proxy_path=$(pwd)/bin/kube-proxy \ - -portal_net=10.10.10.0/24 + -portal_net=10.10.10.0/24 \ + -mesos_user=root ``` For simpler execution of `kubecfg`: