From 47b52d272c7e19361ca4c02cb94e3bdb1fd562b8 Mon Sep 17 00:00:00 2001 From: mchtech Date: Sat, 4 Aug 2018 01:23:31 +0800 Subject: [PATCH] ZStack EIP and PortForwarding Support --- zstack/networkservice.go | 294 +++++++++++++++++++++++++++++++++++++++ zstack/vm.go | 108 +++++++++++++- 2 files changed, 396 insertions(+), 6 deletions(-) create mode 100644 zstack/networkservice.go diff --git a/zstack/networkservice.go b/zstack/networkservice.go new file mode 100644 index 0000000..097950e --- /dev/null +++ b/zstack/networkservice.go @@ -0,0 +1,294 @@ +package zstack + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" + + "github.com/Sirupsen/logrus" + "github.com/cnrancher/go-zstack/common" + "github.com/pkg/errors" +) + +const ( + vipURI = "/zstack/v1/vips" + deleteVipURI = "/zstack/v1/vips/{uuid}" + deleteInstanceURI = "/zstack/v1/vm-instances/{uuid}" + portForwardRuleURI = "/zstack/v1/port-forwarding" + eipURI = "/zstack/v1/eips" +) + +//CreatePortForwardRuleRequest struct +type CreatePortForwardRuleRequest struct { + CreatePortForwardRuleContent map[string]string `json:"params,omitempty"` + Tags common.Tags `json:",inline"` +} + +//PortForwardRule struct +type PortForwardRule struct { + UUID string `json:"uuid,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + VIPIP string `json:"vipIp,omitempty"` + GuestIP string `json:"guestIp,omitempty"` + VipUUID string `json:"vipUuid,omitempty"` + VipPortStart uint16 `json:"vipPortStart,omitempty"` + VipPortEnd uint16 `json:"vipPortEnd,omitempty"` + PrivatePortStart uint16 `json:"privatePortStart,omitempty"` + PrivatePortEnd uint16 `json:"privatePortEnd,omitempty"` + VMNicUUID string `json:"vmNicUuid,omitempty"` + ProtocolType string `json:"protocolType,omitempty"` + State string `json:"state,omitempty"` + AllowedCidr string `json:"allowedCidr,omitempty"` + CreateDate string `json:"createDate,omitempty"` + LastOpDate string `json:"lastOpDate,omitempty"` +} + +//CreatePortForwardRuleResponse struct +type CreatePortForwardRuleResponse struct { + Error *common.Error `json:"error,omitempty"` + Inventory PortForwardRule `json:"inventory,omitempty"` +} + +//QueryPortForwardRulesResponse struct +type QueryPortForwardRulesResponse struct { + Error *common.Error `json:"error,omitempty"` + Inventories []PortForwardRule `json:"inventories,omitempty"` +} + +//CreateVipRequest struct +type CreateVipRequest struct { + CreateVipContent map[string]string `json:"params,omitempty"` + Tags common.Tags `json:",inline"` +} + +//VipRule struct +type VipRule struct { + UUID string `json:"uuid,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + L3NetworkUUID string `json:"l3NetworkUuid,omitempty"` + IP string `json:"ip,omitempty"` + State string `json:"state,omitempty"` + Gateway string `json:"gateway,omitempty"` + Netmask string `json:"netmask,omitempty"` + ServiceProvider string `json:"serviceProvider,omitempty"` + PeerL3NetworkUUID []string `json:"peerL3NetworkUuid,omitempty"` + UseFor string `json:"useFor,omitempty"` + CreateDate string `json:"createDate,omitempty"` + LastOpDate string `json:"lastOpDate,omitempty"` +} + +//CreateVipResponse struct +type CreateVipResponse struct { + Error *common.Error `json:"error,omitempty"` + Inventory VipRule `json:"inventory,omitempty"` +} + +//VipResponses struct +type VipResponses struct { + Error *common.Error `json:"error,omitempty"` + Inventories []VipRule `json:"inventories,omitempty"` +} + +//CreateEipRequest struct +type CreateEipRequest struct { + CreateEipContent map[string]string `json:"params,omitempty"` + Tags common.Tags `json:",inline"` +} + +//CreateEipResponse struct +type CreateEipResponse struct { + Error *common.Error `json:"error,omitempty"` + Inventory struct { + UUID string `json:"uuid,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + VMNicUUID string `json:"vmNicUuid,omitempty"` + VipUUID string `json:"vipUuid,omitempty"` + CreateDate string `json:"createDate,omitempty"` + LastOpDate string `json:"lastOpDate,omitempty"` + State string `json:"state,omitempty"` + VipIP string `json:"vipIp,omitempty"` + GuestIP string `json:"guestIp,omitempty"` + } `json:"inventory,omitempty"` +} + +//CreateEip will map created public vip to vm private ip +func (d *Driver) CreateEip(vipuuid string, vmnicuuid string) error { + d.initClients() + eip := CreateEipRequest{ + CreateEipContent: map[string]string{ + "name": "eip-for-rancher-" + d.InstanceUUID, + "vipUuid": vipuuid, + "vmNicUuid": vmnicuuid, + }, + Tags: common.Tags{ + SystemTags: []string{}, + UserTags: []string{}, + }, + } + requestBody, _ := json.Marshal(eip) + resp, err := d.getInstanceClient().CreateRequestWithURI(http.MethodPost, eipURI, requestBody) + + if err != nil { + return err + } + + async, err := common.GetAsyncResponse(d.client, resp) + responseStruct := CreateEipResponse{} + + if err = async.QueryRealResponse(&responseStruct, 60*time.Second); err != nil { + return errors.Wrap(err, "Get error when query response for zstack create EIP job.") + } + if responseStruct.Error != nil { + return errors.Wrap(responseStruct.Error.WrapError(), "Get error when create zstack EIP.") + } + return nil +} + +//CreateVip will create a public vip from provided public L3 network pool +func (d *Driver) CreateVip() (string, string, error) { + d.initClients() + vip := CreateVipRequest{ + CreateVipContent: map[string]string{ + "name": "vip-for-rancher-" + d.InstanceUUID, + "l3NetworkUuid": d.PublicL3NetworkUUID, + }, + Tags: common.Tags{ + SystemTags: []string{}, + UserTags: []string{}, + }, + } + requestBody, _ := json.Marshal(vip) + + resp, err := d.getInstanceClient().CreateRequestWithURI(http.MethodPost, vipURI, requestBody) + + if err != nil { + return "", "", err + } + + async, err := common.GetAsyncResponse(d.client, resp) + responseStruct := CreateVipResponse{} + + if err = async.QueryRealResponse(&responseStruct, 60*time.Second); err != nil { + return "", "", errors.Wrap(err, "Get error when query response for zstack create VIP job.") + } + if responseStruct.Error != nil { + return "", "", errors.Wrap(responseStruct.Error.WrapError(), "Get error when delete zstack VIP.") + } + return responseStruct.Inventory.IP, responseStruct.Inventory.UUID, nil +} + +//DeleteVip will delete public vip and related EIP and/or Portforwarding rules +func (d *Driver) DeleteVip() error { + d.initClients() + // _, publicipv4uuid, err := d.QueryVipIPUUID() + // if err != nil { + // return err + // } + + if d.PublicIPv4UUID == "" { + return errors.New("PublicIPv4UUID is empty") + } + + realURI := strings.Replace(deleteVipURI, "{uuid}", d.PublicIPv4UUID, -1) + resp, err := d.getInstanceClient().CreateRequestWithURI(http.MethodDelete, realURI, nil) + if err != nil { + return err + } + + async, err := common.GetAsyncResponse(d.client, resp) + responseStruct := CreateVipResponse{} + + if err = async.QueryRealResponse(&responseStruct, 60*time.Second); err != nil { + return errors.Wrap(err, "Get error when query response for zstack delete vip job.") + } + if responseStruct.Error != nil { + return errors.Wrap(responseStruct.Error.WrapError(), "Get error when delete zstack vip.") + } + return nil +} + +//CreatePortForwardRule will create a tcp or udp port forward rule from port 1 to 65535 +func (d *Driver) CreatePortForwardRule(vipuuid string, vmnicuuid string, proto string) error { + d.initClients() + portforward := CreatePortForwardRuleRequest{ + CreatePortForwardRuleContent: map[string]string{ + "vipUuid": vipuuid, + "vipPortStart": "1", + "vipPortEnd": "65535", + "privatePortStart": "1", + "privatePortEnd": "65535", + "protocolType": proto, + "vmNicUuid": vmnicuuid, + "name": "pf-for-rancher-" + proto + "-" + d.InstanceUUID, + }, + Tags: common.Tags{ + SystemTags: []string{}, + UserTags: []string{}, + }, + } + requestBody, _ := json.Marshal(portforward) + resp, err := d.getInstanceClient().CreateRequestWithURI(http.MethodPost, portForwardRuleURI, requestBody) + if err != nil { + return err + } + responseStruct := CreatePortForwardRuleResponse{} + async, err := common.GetAsyncResponse(d.client, resp) + + if err = async.QueryRealResponse(&responseStruct, 60*time.Second); err != nil { + return errors.Wrap(err, "Get error when query response for zstack create portforward rules.") + } + if responseStruct.Error != nil { + return errors.Wrap(responseStruct.Error.WrapError(), "Get error when create portforward rules.") + } + return nil +} + +//QueryVipIPUUID will return vip and its uuid of current instance +func (d *Driver) QueryVipIPUUID() (string, string, error) { + d.initClients() + inventory, err := d.getInstanceClient().QueryInstance(d.InstanceUUID) + if err != nil { + return "", "", err + } + if len(inventory.VMNics) == 0 { + return "", "", fmt.Errorf("Nics not found") + } + GuestIP := inventory.VMNics[0].IP + var realURI string + + if strings.ToLower(d.PublicL3NetworkMode) == "portforward" { + realURI = fmt.Sprintf("%s?q=portForwarding.guestIp=%s", vipURI, GuestIP) + } else if strings.ToLower(d.PublicL3NetworkMode) == "eip" { + realURI = fmt.Sprintf("%s?q=eip.guestIp=%s", vipURI, GuestIP) + } else { + return "", "", fmt.Errorf("Unsupported network mode") + } + resp, err := d.getInstanceClient().CreateRequestWithURI(http.MethodGet, realURI, nil) + if err != nil { + return "", "", err + } + responseBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", "", err + } + responseStruct := VipResponses{} + if err = json.Unmarshal(responseBody, &responseStruct); err != nil { + logrus.Warnf("Unmarshaling response when Querying pf rules. Error: %s", err.Error()) + } + if resp.StatusCode != 200 { + if responseStruct.Error != nil { + return "", "", responseStruct.Error.WrapError() + } + return "", "", fmt.Errorf("status code %d,Error massage %s", resp.StatusCode, string(responseBody)) + } + if len(responseStruct.Inventories) > 0 { + return responseStruct.Inventories[0].IP, responseStruct.Inventories[0].UUID, nil + } + return "", "", fmt.Errorf("Not any portforward rules") +} diff --git a/zstack/vm.go b/zstack/vm.go index 471aec4..a9234b1 100644 --- a/zstack/vm.go +++ b/zstack/vm.go @@ -32,6 +32,7 @@ const ( // return &Driver{} //} +//NewDriver : func NewDriver(hostName, storePath string) drivers.Driver { return &Driver{ BaseDriver: &drivers.BaseDriver{ @@ -41,6 +42,7 @@ func NewDriver(hostName, storePath string) drivers.Driver { }} } +//Driver : ZStack docker-machine driver struct type Driver struct { *drivers.BaseDriver AccountName string @@ -61,6 +63,13 @@ type Driver struct { L3NetworkNames string + PublicL3NetworkUUID string + PublicIPv4 string + PublicIPv4UUID string + PrivateIPv4 string + PrivateIPAddress string + PublicL3NetworkMode string + SystemDiskOffering string DataDiskOffering string @@ -71,6 +80,7 @@ type Driver struct { InstanceUUID string + client *common.Client instanceClient *instance.Client hostClient *infrastructure.Host imageClient *instance.Image @@ -83,6 +93,7 @@ type Driver struct { func (d *Driver) cleanup() error { defer func() { + d.client = nil d.hostClient = nil d.imageClient = nil d.clusterClient = nil @@ -104,6 +115,8 @@ func (d *Driver) initClients() error { log.Error(err) return err } + d.client = &commonClient + d.instanceClient = &instance.Client{ Client: commonClient, } @@ -142,7 +155,6 @@ func (d *Driver) getInstanceClient() *instance.Client { // Create a host using the driver's config func (d *Driver) Create() error { - var ( err error ) @@ -162,7 +174,7 @@ func (d *Driver) Create() error { request.Params.DataDiskOfferingUUIDs = d.getDataDisks() request.Params.PrimaryStorageUUIDForRootVolume = d.PrimaryStorage request.Params.HostUUID = d.PhysicalHost - async, err := d.instanceClient.CreateInstance(request) + async, err := d.getInstanceClient().CreateInstance(request) if err != nil { return errors.Wrap(err, "Get error when create vm instance in zstack.") } @@ -174,12 +186,49 @@ func (d *Driver) Create() error { return errors.Wrap(response.Error.WrapError(), "Get error when create vm instance in zstack.") } d.InstanceUUID = response.Inventory.UUID - - inventory, err := d.instanceClient.QueryInstance(d.InstanceUUID) + inventory, err := d.getInstanceClient().QueryInstance(d.InstanceUUID) if err != nil { return err } + //set up public IP map + if d.PublicL3NetworkUUID != "" { + if len(inventory.VMNics) > 0 { + + if strings.ToLower(d.PublicL3NetworkMode) == "portforward" { + d.PublicIPv4, d.PublicIPv4UUID, err = d.CreateVip() + if err != nil { + return err + } + for _, proto := range []string{"TCP", "UDP"} { + err = d.CreatePortForwardRule(d.PublicIPv4UUID, inventory.VMNics[0].UUID, proto) + if err != nil { + log.Errorf("error create portforwarding rules: %s", err.Error()) + return err + } + } + } else if strings.ToLower(d.PublicL3NetworkMode) == "eip" { + d.PublicIPv4, d.PublicIPv4UUID, err = d.CreateVip() + if err != nil { + return err + } + err = d.CreateEip(d.PublicIPv4UUID, inventory.VMNics[0].UUID) + if err != nil { + log.Errorf("error create eip: %s", err.Error()) + return err + } + } else { + + } + } + } + d.IPAddress = d.getIP(inventory) + d.PublicIPv4 = d.IPAddress + d.PrivateIPAddress, _ = d.GetInternalIP() + d.PrivateIPv4 = d.PrivateIPAddress + + log.Debugf("IP Address: %s, %s; %s, %s.", d.PublicIPv4, d.IPAddress, d.PrivateIPv4, d.PrivateIPAddress) + if d.SSHUser == "" { d.SSHUser = sshUser } @@ -264,6 +313,7 @@ func (d *Driver) autoFdisk(sshClient ssh.Client) { func (d *Driver) configInstance() error { ipAddr := d.IPAddress + //ipAddr := d.PrivateIPAddress port, _ := d.GetSSHPort() tcpAddr := fmt.Sprintf("%s:%d", ipAddr, port) @@ -397,6 +447,18 @@ func (d *Driver) GetCreateFlags() []mcnflag.Flag { EnvVar: "ZSTACK_SSH_PASSWORD", Value: "", }, + mcnflag.StringFlag{ + Name: "zstack-public-network", + Usage: "Optional. Specify the public network uuid for EIP or PortForward mode.", + EnvVar: "ZSTACK_PUBLIC_NETWORK", + Value: "", + }, + mcnflag.StringFlag{ + Name: "zstack-network-mode", + Usage: "EIP, PortForward, Flat (Default)", + EnvVar: "ZSTACK_NETWORK_MODE", + Value: "", + }, } } @@ -413,12 +475,14 @@ func (d *Driver) GetIP() (string, error) { // GetSSHHostname returns hostname for use with ssh func (d *Driver) GetSSHHostname() (string, error) { return d.GetIP() + //return d.GetInternalIP() } // GetURL returns a Docker compatible host URL for connecting to this host // e.g. tcp://1.2.3.4:2376 func (d *Driver) GetURL() (string, error) { ip, err := d.GetIP() + //ip, err := d.GetInternalIP() if err != nil { return "", err } @@ -459,7 +523,7 @@ func (d *Driver) GetState() (state.State, error) { // Kill stops a host forcefully func (d *Driver) Kill() error { - async, err := d.instanceClient.StopInstance(d.InstanceUUID, instance.StopInstanceTypeCold) + async, err := d.getInstanceClient().StopInstance(d.InstanceUUID, instance.StopInstanceTypeCold) if err != nil { return errors.Wrap(err, "Get error when sending kill instance request.") } @@ -490,6 +554,11 @@ func (d *Driver) PreCreateCheck() error { // Remove a host func (d *Driver) Remove() error { + //Zero delete vip (automatically remove portforward rules and eip) + if d.PublicL3NetworkUUID != "" && (strings.ToLower(d.PublicL3NetworkMode) == "eip" || strings.ToLower(d.PublicL3NetworkMode) == "portforward") { + d.DeleteVip() + } + //First delete it async, err := d.getInstanceClient().DeleteInstance(d.InstanceUUID) if err != nil { @@ -515,7 +584,6 @@ func (d *Driver) Remove() error { if responseStruct.Error != nil { return errors.Wrap(responseStruct.Error.WrapError(), "Get error when expunge zstack instance.") } - return nil } @@ -561,6 +629,14 @@ func (d *Driver) SetConfigFromFlags(opts drivers.DriverOptions) error { return errors.Errorf("The network configuration is required.") } + d.PublicL3NetworkUUID = opts.String("zstack-public-network") + d.PublicL3NetworkMode = opts.String("zstack-network-mode") + if d.PublicL3NetworkMode == "" { + return errors.Errorf("The L3 network mode is required.") + } + if strings.ToLower(d.PublicL3NetworkMode) != "flat" && d.PublicL3NetworkUUID == "" { + return errors.Errorf("The public L3 network UUID is required when network mode is not FlatNetwork.") + } //if the image is the type of ISO, then this argument is required d.SystemDiskOffering = opts.String("zstack-system-disk-offering") if d.SystemDiskOffering == "" { @@ -618,8 +694,28 @@ func (d *Driver) Stop() error { } func (d *Driver) getIP(instance *instance.VMInstanceInventory) string { + //log.Infof("Try to return PublicIPv4: %s", d.PublicIPv4) + if d.PublicIPv4 != "" { + return d.PublicIPv4 + } + // PublicIPv4, _, _ := d.QueryVipIPUUID() + // if PublicIPv4 != "" { + // return PublicIPv4 + // } if len(instance.VMNics) > 0 { return instance.VMNics[0].IP } return "" } + +//GetInternalIP will get vm private ip rather than public ip +func (d *Driver) GetInternalIP() (string, error) { + inventory, err := d.getInstanceClient().QueryInstance(d.InstanceUUID) + if err != nil { + return "", errors.Wrap(err, "Error when getting instance.") + } + if len(inventory.VMNics) > 0 { + return inventory.VMNics[0].IP, nil + } + return "", errors.Errorf("IP not found") +}