diff --git a/cmd/kubestack/kubestack.go b/cmd/kubestack/kubestack.go index 73e7847..b563a78 100644 --- a/cmd/kubestack/kubestack.go +++ b/cmd/kubestack/kubestack.go @@ -75,7 +75,7 @@ func (os *OpenStack) getNetworkIDByNamespace(namespace string) (string, error) { // Only support one network and network's name is same with namespace. // TODO: make it general after multi-network is supported. networkName := util.BuildNetworkName(namespace, namespace) - network, err := os.Client.GetNetwork(networkName) + network, err := os.Client.GetNetworkByName(networkName) if err != nil { glog.Errorf("Get network by name %q failed: %v", networkName, err) return "", err diff --git a/pkg/network-controller/network_controller_helper.go b/pkg/network-controller/network_controller_helper.go index a93b8df..4ecaf99 100644 --- a/pkg/network-controller/network_controller_helper.go +++ b/pkg/network-controller/network_controller_helper.go @@ -82,8 +82,8 @@ func (c *NetworkController) addNetworkToDriver(kubeNetwork *crv1.Network) error glog.V(4).Infof("[NetworkController]: adding network %s", driverNetwork.Name) - // Check if tenant id exist - check, err := c.driver.CheckTenantID(driverNetwork.TenantID) + // Check if tenant exist or not by tenantID. + check, err := c.driver.CheckTenantByID(driverNetwork.TenantID) if err != nil { glog.Errorf("[NetworkController]: check tenantID failed: %v", err) return err @@ -107,7 +107,7 @@ func (c *NetworkController) addNetworkToDriver(kubeNetwork *crv1.Network) error newNetworkStatus = crv1.NetworkFailed } else { // Check if provider network has already created - _, err := c.driver.GetNetwork(networkName) + _, err := c.driver.GetNetworkByName(networkName) if err == nil { glog.Infof("[NetworkController]: network %s has already created", networkName) } else if err.Error() == util.ErrNotFound.Error() { diff --git a/pkg/openstack/client.go b/pkg/openstack/client.go index 05c2697..072cda4 100644 --- a/pkg/openstack/client.go +++ b/pkg/openstack/client.go @@ -71,8 +71,8 @@ type Interface interface { DeleteTenant(tenantName string) error // GetTenantIDFromName gets tenantID by tenantName. GetTenantIDFromName(tenantName string) (string, error) - // CheckTenantID checks tenant exist or not. - CheckTenantID(tenantID string) (bool, error) + // CheckTenantByID checks tenant exist or not by tenantID. + CheckTenantByID(tenantID string) (bool, error) // CreateUser creates user with username, password in the tenant. CreateUser(username, password, tenantID string) error // DeleteAllUsersOnTenant deletes all users on the tenant. @@ -81,8 +81,8 @@ type Interface interface { CreateNetwork(network *drivertypes.Network) error // GetNetworkByID gets network by networkID. GetNetworkByID(networkID string) (*drivertypes.Network, error) - // GetNetwork gets networks by networkName. - GetNetwork(networkName string) (*drivertypes.Network, error) + // GetNetworkByName gets network by networkName. + GetNetworkByName(networkName string) (*drivertypes.Network, error) // DeleteNetwork deletes network by networkName. DeleteNetwork(networkName string) error // GetProviderSubnet gets provider subnet by id @@ -235,6 +235,7 @@ func (os *Client) GetIntegrationBridge() string { return os.IntegrationBridge } +// GetTenantIDFromName gets tenantID by tenantName. func (os *Client) GetTenantIDFromName(tenantName string) (string, error) { if util.IsSystemNamespace(tenantName) { tenantName = util.SystemTenant @@ -276,6 +277,7 @@ func (os *Client) GetTenantIDFromName(tenantName string) (string, error) { return tenantID, nil } +// CreateTenant creates tenant by tenantname. func (os *Client) CreateTenant(tenantName string) (string, error) { createOpts := tenants.CreateOpts{ Name: tenantName, @@ -296,6 +298,7 @@ func (os *Client) CreateTenant(tenantName string) (string, error) { return tenantID, nil } +// DeleteTenant deletes tenant by tenantName. func (os *Client) DeleteTenant(tenantName string) error { return tenants.List(os.Identity, nil).EachPage(func(page pagination.Page) (bool, error) { tenantList, err := tenants.ExtractTenants(page) @@ -317,6 +320,7 @@ func (os *Client) DeleteTenant(tenantName string) error { }) } +// CreateUser creates user with username, password in the tenant. func (os *Client) CreateUser(username, password, tenantID string) error { opts := users.CreateOpts{ Name: username, @@ -333,6 +337,7 @@ func (os *Client) CreateUser(username, password, tenantID string) error { return nil } +// DeleteAllUsersOnTenant deletes all users on the tenant. func (os *Client) DeleteAllUsersOnTenant(tenantName string) error { tenantID, err := os.GetTenantIDFromName(tenantName) if err != nil { @@ -368,7 +373,7 @@ func reasonForError(err error) int { return 0 } -// Get tenant's network by tenantID(tenant and network are one to one mapping in stackube) +// GetOpenStackNetworkByTenantID gets tenant's network by tenantID(tenant and network are one to one mapping in stackube) func (os *Client) GetOpenStackNetworkByTenantID(tenantID string) (*networks.Network, error) { opts := networks.ListOpts{TenantID: tenantID} return os.getOpenStackNetwork(&opts) @@ -410,7 +415,7 @@ func (os *Client) getOpenStackNetwork(opts *networks.ListOpts) (*networks.Networ return osNetwork, err } -// Get provider subnet by id +// GetProviderSubnet gets provider subnet by subnetID func (os *Client) GetProviderSubnet(osSubnetID string) (*drivertypes.Subnet, error) { s, err := subnets.Get(os.Network, osSubnetID).Extract() if err != nil { @@ -439,7 +444,7 @@ func (os *Client) GetProviderSubnet(osSubnetID string) (*drivertypes.Subnet, err return &providerSubnet, nil } -// Get network by networkID +// GetNetworkByID gets network by networkID func (os *Client) GetNetworkByID(networkID string) (*drivertypes.Network, error) { osNetwork, err := os.getOpenStackNetworkByID(networkID) if err != nil { @@ -450,8 +455,8 @@ func (os *Client) GetNetworkByID(networkID string) (*drivertypes.Network, error) return os.OSNetworktoProviderNetwork(osNetwork) } -// Get network by networkName -func (os *Client) GetNetwork(networkName string) (*drivertypes.Network, error) { +// GetNetworkByName gets network by networkName +func (os *Client) GetNetworkByName(networkName string) (*drivertypes.Network, error) { osNetwork, err := os.getOpenStackNetworkByName(networkName) if err != nil { glog.Warningf("try to fetch openstack network by name: %v but failed: %v", networkName, err) @@ -495,7 +500,7 @@ func (os *Client) ToProviderStatus(status string) string { } } -// Create network +// CreateNetwork creates network. func (os *Client) CreateNetwork(network *drivertypes.Network) error { if len(network.Subnets) == 0 { return errors.New("Subnets is null") @@ -573,7 +578,7 @@ func (os *Client) CreateNetwork(network *drivertypes.Network) error { return nil } -// Update network +// UpdateNetwork updates network. func (os *Client) UpdateNetwork(network *drivertypes.Network) error { // TODO: update network subnets return nil @@ -601,7 +606,7 @@ func (os *Client) getRouterByName(name string) (*routers.Router, error) { return result, nil } -// Delete network by networkName +// DeleteNetwork deletes network by networkName. func (os *Client) DeleteNetwork(networkName string) error { osNetwork, err := os.getOpenStackNetworkByName(networkName) if err != nil { @@ -681,8 +686,8 @@ func (os *Client) DeleteNetwork(networkName string) error { return nil } -// Check the tenant id exist -func (os *Client) CheckTenantID(tenantID string) (bool, error) { +// CheckTenantByID checks tenant exist or not by tenantID. +func (os *Client) CheckTenantByID(tenantID string) (bool, error) { opts := tenants.ListOpts{} pager := tenants.List(os.Identity, &opts) @@ -710,6 +715,7 @@ func (os *Client) CheckTenantID(tenantID string) (bool, error) { return found, err } +// GetPort gets port by portName. func (os *Client) GetPort(name string) (*ports.Port, error) { opts := ports.ListOpts{Name: name} pager := ports.List(os.Network, opts) @@ -831,7 +837,7 @@ func (os *Client) ensureSecurityGroup(tenantID string) (string, error) { return securitygroup.ID, nil } -// Create an port +// CreatePort creates port by neworkID, tenantID and portName. func (os *Client) CreatePort(networkID, tenantID, portName string) (*portsbinding.Port, error) { securitygroup, err := os.ensureSecurityGroup(tenantID) if err != nil { @@ -860,7 +866,7 @@ func (os *Client) CreatePort(networkID, tenantID, portName string) (*portsbindin return port, nil } -// List all ports in the network +// ListPorts lists ports by networkID and deviceOwner. func (os *Client) ListPorts(networkID, deviceOwner string) ([]ports.Port, error) { var results []ports.Port opts := ports.ListOpts{ diff --git a/pkg/openstack/openstack_fake.go b/pkg/openstack/openstack_fake.go new file mode 100644 index 0000000..0057c9b --- /dev/null +++ b/pkg/openstack/openstack_fake.go @@ -0,0 +1,400 @@ +/* +Copyright (c) 2017 OpenStack Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + "crypto/sha1" + "errors" + "fmt" + "io" + + crv1 "git.openstack.org/openstack/stackube/pkg/apis/v1" + crdClient "git.openstack.org/openstack/stackube/pkg/kubecrd" + drivertypes "git.openstack.org/openstack/stackube/pkg/openstack/types" + "git.openstack.org/openstack/stackube/pkg/util" + "github.com/gophercloud/gophercloud/openstack/identity/v2/tenants" + "github.com/gophercloud/gophercloud/openstack/identity/v2/users" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/portsbinding" + "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" + "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" +) + +var ErrAlreadyExist = errors.New("AlreadyExist") + +// FakeOSClient is a simple fake openstack client, so that stackube +// can be run for testing without requiring a real openstack setup. +type FakeOSClient struct { + Tenants map[string]*tenants.Tenant + Users map[string]*users.User + Networks map[string]*drivertypes.Network + Subnets map[string]*subnets.Subnet + Routers map[string]*routers.Router + Ports map[string][]ports.Port + // TODO(mozhuli): Add fakeCRDClient. + CRDClient *crdClient.CRDClient + PluginName string + IntegrationBridge string +} + +var _ = Interface(&FakeOSClient{}) + +// NewFake creates a new FakeOSClient. +func NewFake() *FakeOSClient { + return &FakeOSClient{ + Tenants: make(map[string]*tenants.Tenant), + Users: make(map[string]*users.User), + Networks: make(map[string]*drivertypes.Network), + Subnets: make(map[string]*subnets.Subnet), + Routers: make(map[string]*routers.Router), + Ports: make(map[string][]ports.Port), + PluginName: "ovs", + IntegrationBridge: "bi-int", + } +} + +// SetTenant injects fake tenant. +func (os *FakeOSClient) SetTenant(tenantName, tenantID string) { + tenant := &tenants.Tenant{ + Name: tenantName, + ID: tenantID, + } + os.Tenants[tenantName] = tenant +} + +// SetUser injects fake user. +func (os *FakeOSClient) SetUser(userName, userID, tenantID string) { + user := &users.User{ + Username: userName, + ID: userID, + TenantID: tenantID, + } + os.Users[tenantID] = user +} + +// SetNetwork injects fake network. +func (os *FakeOSClient) SetNetwork(networkName, networkID string) { + network := &drivertypes.Network{ + Name: networkName, + Uid: networkID, + } + os.Networks[networkName] = network +} + +// SetPort injects fake port. +func (os *FakeOSClient) SetPort(networkID, deviceOwner, deviceID string) { + netPorts, ok := os.Ports[networkID] + p := ports.Port{ + NetworkID: networkID, + DeviceOwner: deviceOwner, + DeviceID: deviceID, + } + if !ok { + var ps []ports.Port + ps = append(ps, p) + os.Ports[networkID] = ps + } + netPorts = append(netPorts, p) + os.Ports[networkID] = netPorts +} + +func tenantIDHash(tenantName string) string { + return idHash(tenantName) +} + +func userIDHash(userName, tenantID string) string { + return idHash(userName) +} + +func networkIDHash(networkName string) string { + return idHash(networkName) +} + +func subnetIDHash(subnetName string) string { + return idHash(subnetName) +} + +func routerIDHash(routerName string) string { + return idHash(routerName) +} + +func portdeviceIDHash(networkID, deviceOwner string) string { + return idHash(networkID, deviceOwner) +} + +func idHash(data ...string) string { + var s string + for _, d := range data { + s += d + } + h := sha1.New() + io.WriteString(h, s) + return fmt.Sprintf("%x", h.Sum(nil))[:16] +} + +// CreateTenant creates tenant by tenantname. +func (os *FakeOSClient) CreateTenant(tenantName string) (string, error) { + if t, ok := os.Tenants[tenantName]; ok { + return t.ID, nil + } + tenant := &tenants.Tenant{ + Name: tenantName, + ID: tenantIDHash(tenantName), + } + os.Tenants[tenantName] = tenant + return tenant.ID, nil +} + +// DeleteTenant deletes tenant by tenantName. +func (os *FakeOSClient) DeleteTenant(tenantName string) error { + delete(os.Tenants, tenantName) + return nil +} + +// GetTenantIDFromName gets tenantID by tenantName. +func (os *FakeOSClient) GetTenantIDFromName(tenantName string) (string, error) { + if util.IsSystemNamespace(tenantName) { + tenantName = util.SystemTenant + } + + // If tenantID is specified, return it directly + var ( + tenant *crv1.Tenant + err error + ) + // TODO(mozhuli): use fakeCRDClient. + if tenant, err = os.CRDClient.GetTenant(tenantName); err != nil { + return "", err + } + if tenant.Spec.TenantID != "" { + return tenant.Spec.TenantID, nil + } + + t, ok := os.Tenants[tenantName] + if !ok { + return "", nil + } + + return t.ID, nil +} + +// CheckTenantByID checks tenant exist or not by tenantID. +func (os *FakeOSClient) CheckTenantByID(tenantID string) (bool, error) { + for _, tenent := range os.Tenants { + if tenent.ID == tenantID { + return true, nil + } + } + return false, ErrNotFound +} + +// CreateUser creates user with username, password in the tenant. +func (os *FakeOSClient) CreateUser(username, password, tenantID string) error { + user := &users.User{ + Name: username, + TenantID: tenantID, + ID: userIDHash(username, tenantID), + } + os.Users[tenantID] = user + return nil +} + +// DeleteAllUsersOnTenant deletes all users on the tenant. +func (os *FakeOSClient) DeleteAllUsersOnTenant(tenantName string) error { + tenant := os.Tenants[tenantName] + + delete(os.Users, tenant.ID) + return nil +} + +func (os *FakeOSClient) createNetwork(networkName, tenantID string) error { + if _, ok := os.Networks[networkName]; ok { + return ErrAlreadyExist + } + + network := &drivertypes.Network{ + Name: networkName, + Uid: networkIDHash(networkName), + TenantID: tenantID, + } + os.Networks[networkName] = network + return nil +} + +func (os *FakeOSClient) deleteNetwork(networkName string) error { + delete(os.Networks, networkName) + return nil +} + +func (os *FakeOSClient) createRouter(routerName, tenantID string) error { + if _, ok := os.Routers[routerName]; ok { + return ErrAlreadyExist + } + + router := &routers.Router{ + Name: routerName, + TenantID: tenantID, + ID: routerIDHash(routerName), + } + os.Routers[routerName] = router + return nil +} + +func (os *FakeOSClient) deleteRouter(routerName string) error { + delete(os.Routers, routerName) + return nil +} + +func (os *FakeOSClient) createSubnet(subnetName, networkID, tenantID string) error { + if _, ok := os.Subnets[subnetName]; ok { + return ErrAlreadyExist + } + + subnet := &subnets.Subnet{ + Name: subnetName, + TenantID: tenantID, + NetworkID: networkID, + ID: subnetIDHash(subnetName), + } + os.Subnets[subnetName] = subnet + return nil +} + +// CreateNetwork creates network. +// TODO(mozhuli): make it more general. +func (os *FakeOSClient) CreateNetwork(network *drivertypes.Network) error { + if len(network.Subnets) == 0 { + return errors.New("Subnets is null") + } + + // create network + err := os.createNetwork(network.Name, network.TenantID) + if err != nil { + return errors.New("Create network failed") + } + // create router, and use network name as router name for convenience. + err = os.createRouter(network.Name, network.TenantID) + if err != nil { + os.deleteNetwork(network.Name) + return errors.New("Create router failed") + } + // create subnets and connect them to router + err = os.createSubnet(network.Subnets[0].Name, network.Uid, network.TenantID) + if err != nil { + os.deleteRouter(network.Name) + os.deleteNetwork(network.Name) + return errors.New("Create subnet failed") + } + return nil +} + +// GetNetworkByID gets network by networkID. +func (os *FakeOSClient) GetNetworkByID(networkID string) (*drivertypes.Network, error) { + return nil, nil +} + +// GetNetworkByName get network by networkName +func (os *FakeOSClient) GetNetworkByName(networkName string) (*drivertypes.Network, error) { + network, ok := os.Networks[networkName] + if !ok { + return nil, ErrNotFound + } + + return network, nil +} + +// DeleteNetwork deletes network by networkName. +func (os *FakeOSClient) DeleteNetwork(networkName string) error { + return nil +} + +// GetProviderSubnet gets provider subnet by id +func (os *FakeOSClient) GetProviderSubnet(osSubnetID string) (*drivertypes.Subnet, error) { + return nil, nil +} + +// CreatePort creates port by neworkID, tenantID and portName. +func (os *FakeOSClient) CreatePort(networkID, tenantID, portName string) (*portsbinding.Port, error) { + return nil, nil +} + +// GetPort gets port by portName. +func (os *FakeOSClient) GetPort(name string) (*ports.Port, error) { + return nil, nil +} + +// ListPorts list all ports which have the deviceOwner in the network. +func (os *FakeOSClient) ListPorts(networkID, deviceOwner string) ([]ports.Port, error) { + var results []ports.Port + portList, ok := os.Ports[networkID] + if !ok { + return results, nil + } + + for _, port := range portList { + if port.DeviceOwner == deviceOwner { + results = append(results, port) + } + } + return results, nil +} + +// DeletePortByName deletes port by portName. +func (os *FakeOSClient) DeletePortByName(portName string) error { + return nil +} + +// DeletePortByID deletes port by portID. +func (os *FakeOSClient) DeletePortByID(portID string) error { + return nil +} + +// UpdatePortsBinding updates port binding. +func (os *FakeOSClient) UpdatePortsBinding(portID, deviceOwner string) error { + return nil +} + +// LoadBalancerExist returns whether a load balancer has already been exist. +func (os *FakeOSClient) LoadBalancerExist(name string) (bool, error) { + return true, nil +} + +// EnsureLoadBalancer ensures a load balancer is created. +func (os *FakeOSClient) EnsureLoadBalancer(lb *LoadBalancer) (*LoadBalancerStatus, error) { + return nil, nil +} + +// EnsureLoadBalancerDeleted ensures a load balancer is deleted. +func (os *FakeOSClient) EnsureLoadBalancerDeleted(name string) error { + return nil +} + +// GetCRDClient returns the CRDClient. +// TODO(mozhuli): use fakeCRDClient. +func (os *FakeOSClient) GetCRDClient() *crdClient.CRDClient { + return os.CRDClient +} + +// GetPluginName returns the plugin name. +func (os *FakeOSClient) GetPluginName() string { + return os.PluginName +} + +// GetIntegrationBridge returns the integration bridge name. +func (os *FakeOSClient) GetIntegrationBridge() string { + return os.IntegrationBridge +} diff --git a/pkg/proxy/iptables_fake.go b/pkg/proxy/iptables_fake.go new file mode 100644 index 0000000..ef12b22 --- /dev/null +++ b/pkg/proxy/iptables_fake.go @@ -0,0 +1,99 @@ +/* +Copyright (c) 2017 OpenStack Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "fmt" + "strings" +) + +const ( + Destination = "-d " + Source = "-s " + DPort = "--dport " + Protocol = "-p " + Jump = "-j " + ToDest = "--to-destination " +) + +// Rule represents chain's rule. +type Rule map[string]string + +// FakeIPTables have noop implementation of fake iptables function. +type FakeIPTables struct { + namespace string + NSLines map[string][]byte +} + +// NewFake return new FakeIPTables. +func NewFake() *FakeIPTables { + return &FakeIPTables{ + NSLines: make(map[string][]byte), + } +} + +func (f *FakeIPTables) ensureChain() error { + return nil +} + +func (f *FakeIPTables) ensureRule(op, chain string, args []string) error { + return nil +} + +func (f *FakeIPTables) restoreAll(data []byte) error { + d := make([]byte, len(data)) + copy(d, data) + f.NSLines[f.namespace] = d + return nil +} + +func (f *FakeIPTables) netnsExist() bool { + return true +} + +func (f *FakeIPTables) setNetns(netns string) { + f.namespace = netns +} + +func getToken(line, seperator string) string { + tokens := strings.Split(line, seperator) + if len(tokens) == 2 { + return strings.Split(tokens[1], " ")[0] + } + return "" +} + +// GetRules returns a list of rules for the given chain. +// The chain name must match exactly. +// The matching is pretty dumb, don't rely on it for anything but testing. +func (f *FakeIPTables) GetRules(chainName, namespace string) (rules []Rule) { + for _, l := range strings.Split(string(f.NSLines[namespace]), "\n") { + if strings.Contains(l, fmt.Sprintf("-A %v", chainName)) { + newRule := Rule(map[string]string{}) + for _, arg := range []string{Destination, Source, DPort, Protocol, Jump, ToDest} { + tok := getToken(l, arg) + if tok != "" { + newRule[arg] = tok + } + } + rules = append(rules, newRule) + } + } + return +} + +var _ = iptablesInterface(&FakeIPTables{}) diff --git a/pkg/proxy/proxier.go b/pkg/proxy/proxier.go index ab7cc80..ff75091 100644 --- a/pkg/proxy/proxier.go +++ b/pkg/proxy/proxier.go @@ -201,7 +201,7 @@ func (p *Proxier) getRouterForNamespace(namespace string) (string, error) { // Only support one network and network's name is same with namespace. // TODO: make it general after multi-network is supported. networkName := util.BuildNetworkName(namespace, namespace) - network, err := p.osClient.GetNetwork(networkName) + network, err := p.osClient.GetNetworkByName(networkName) if err != nil { glog.Errorf("Get network by name %q failed: %v", networkName, err) return "", err diff --git a/pkg/proxy/proxier_test.go b/pkg/proxy/proxier_test.go new file mode 100644 index 0000000..e292f32 --- /dev/null +++ b/pkg/proxy/proxier_test.go @@ -0,0 +1,637 @@ +/* +Copyright (c) 2017 OpenStack Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/davecgh/go-spew/spew" + + "git.openstack.org/openstack/stackube/pkg/openstack" + "git.openstack.org/openstack/stackube/pkg/util" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/util/async" +) + +func newFakeServiceInfo(serviceName string, ip net.IP) *serviceInfo { + return &serviceInfo{ + name: serviceName, + clusterIP: ip, + } +} + +func Test_getServiceIP(t *testing.T) { + fp := NewFakeProxier(nil, nil) + + testCases := []struct { + serviceInfo *serviceInfo + expected string + }{{ + // Case[0]: kube-dns service. + serviceInfo: newFakeServiceInfo("kube-dns", net.IPv4(1, 2, 3, 4)), + expected: testclusterDNS, + }, { + // Case[1]: other service. + serviceInfo: newFakeServiceInfo("test", net.IPv4(1, 2, 3, 4)), + expected: "1.2.3.4", + }, + } + + for tci, tc := range testCases { + // outputs + clusterIP := fp.getServiceIP(tc.serviceInfo) + + if clusterIP != tc.expected { + t.Errorf("Case[%d] expected %#v, got %#v", tci, tc.expected, clusterIP) + } + } +} + +const testclusterDNS = "10.20.30.40" + +func NewFakeProxier(ipt iptablesInterface, osClient openstack.Interface) *Proxier { + p := &Proxier{ + clusterDNS: testclusterDNS, + osClient: osClient, + iptables: ipt, + endpointsChanges: newEndpointsChangeMap(""), + serviceChanges: newServiceChangeMap(), + namespaceChanges: newNamespaceChangeMap(), + serviceMap: make(proxyServiceMap), + endpointsMap: make(proxyEndpointsMap), + namespaceMap: make(map[string]*namespaceInfo), + serviceNSMap: make(map[string]proxyServiceMap), + } + + p.syncRunner = async.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, time.Minute, 1) + return p +} + +func hasDNAT(rules []Rule, endpoint string) bool { + for _, r := range rules { + if r[ToDest] == endpoint { + return true + } + } + return false +} + +func makeTestService(namespace, name string, svcFunc func(*v1.Service)) *v1.Service { + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{}, + }, + Spec: v1.ServiceSpec{}, + Status: v1.ServiceStatus{}, + } + svcFunc(svc) + return svc +} + +func makeServiceMap(proxier *Proxier, allServices ...*v1.Service) { + for i := range allServices { + proxier.onServiceAdded(allServices[i]) + } + + proxier.mu.Lock() + defer proxier.mu.Unlock() + proxier.servicesSynced = true +} + +func makeTestEndpoints(namespace, name string, eptFunc func(*v1.Endpoints)) *v1.Endpoints { + ept := &v1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + eptFunc(ept) + return ept +} + +func makeEndpointsMap(proxier *Proxier, allEndpoints ...*v1.Endpoints) { + for i := range allEndpoints { + proxier.onEndpointsAdded(allEndpoints[i]) + } + + proxier.mu.Lock() + defer proxier.mu.Unlock() + proxier.endpointsSynced = true +} + +func makeTestNamespace(name string) *v1.Namespace { + ns := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{}, + }, + Spec: v1.NamespaceSpec{}, + Status: v1.NamespaceStatus{}, + } + return ns +} + +func makeNamespaceMap(proxier *Proxier, allNamespaces ...*v1.Namespace) { + for i := range allNamespaces { + proxier.onNamespaceAdded(allNamespaces[i]) + } + + proxier.mu.Lock() + defer proxier.mu.Unlock() + proxier.namespaceSynced = true +} + +func makeNSN(namespace, name string) types.NamespacedName { + return types.NamespacedName{Namespace: namespace, Name: name} +} + +func makeServicePortName(ns, name, port string) servicePortName { + return servicePortName{ + NamespacedName: makeNSN(ns, name), + Port: port, + } +} + +func errorf(msg string, rules []Rule, t *testing.T) { + for _, r := range rules { + t.Logf("%q", r) + } + t.Errorf("%v", msg) +} + +func TestClusterNoEndpoint(t *testing.T) { + testNamespace := "test" + svcIP := "1.2.3.4" + svcPort := 80 + svcPortName := servicePortName{ + NamespacedName: makeNSN(testNamespace, "svc1"), + Port: "80", + } + + //Creates fake iptables. + ipt := NewFake() + //Create a fake openstack client. + osClient := openstack.NewFake() + // Injects fake network. + networkName := util.BuildNetworkName(testNamespace, testNamespace) + osClient.SetNetwork(networkName, "123") + // Injects fake port. + osClient.SetPort("123", "network:router_interface", "123") + // Creates a new fake proxier. + fp := NewFakeProxier(ipt, osClient) + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.ClusterIP = svcIP + svc.Spec.Type = v1.ServiceTypeNodePort + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + }} + }), + ) + + makeEndpointsMap(fp) + + makeNamespaceMap(fp, makeTestNamespace(svcPortName.Namespace)) + + fp.syncProxyRules() + + stackubeRules := ipt.GetRules(ChainSKPrerouting, "qrouter-123") + if len(stackubeRules) != 0 { + errorf(fmt.Sprintf("Unexpected rule for chain %v without endpoints in namespace %v", ChainSKPrerouting, svcPortName.Namespace), stackubeRules, t) + } +} + +func noClusterIPType(svcType v1.ServiceType) []Rule { + testNamespace := "test" + svcIP := "1.2.3.4" + svcPort := 80 + svcPortName := servicePortName{ + NamespacedName: makeNSN(testNamespace, "svc1"), + Port: "80", + } + + // Creates fake iptables. + ipt := NewFake() + // Create a fake openstack client. + osClient := openstack.NewFake() + // Injects fake network. + networkName := util.BuildNetworkName(testNamespace, testNamespace) + osClient.SetNetwork(networkName, "123") + // Injects fake port. + osClient.SetPort("123", "network:router_interface", "123") + // Creates a new fake proxier. + fp := NewFakeProxier(ipt, osClient) + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Namespace, func(svc *v1.Service) { + svc.Spec.ClusterIP = svcIP + svc.Spec.Type = svcType + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + }} + }), + ) + + makeEndpointsMap(fp) + + makeNamespaceMap(fp, makeTestNamespace(svcPortName.Namespace)) + + fp.syncProxyRules() + + stackubeRules := ipt.GetRules(ChainSKPrerouting, "qrouter-123") + return stackubeRules +} + +func TestNoClusterIPType(t *testing.T) { + testCases := map[string]v1.ServiceType{ + "case 1": v1.ServiceTypeNodePort, + "case 2": v1.ServiceTypeLoadBalancer, + "case 3": v1.ServiceTypeExternalName, + } + + for k, tc := range testCases { + got := noClusterIPType(tc) + if len(got) != 0 { + errorf(fmt.Sprintf("%v: unexpected rule for chain %v without ClusterIP service type", k, ChainSKPrerouting), got, t) + } + } +} + +func TestClusterIPEndpointsJump(t *testing.T) { + testNamespace := "test" + svcIP := "1.2.3.4" + svcPort := 80 + svcPortName := servicePortName{ + NamespacedName: makeNSN(testNamespace, "svc1"), + Port: "80", + } + + // Creates fake iptables. + ipt := NewFake() + // Create a fake openstack client. + osClient := openstack.NewFake() + // Injects fake network. + networkName := util.BuildNetworkName(testNamespace, testNamespace) + osClient.SetNetwork(networkName, "123") + // Injects fake port. + osClient.SetPort("123", "network:router_interface", "123") + // Creates a new fake proxier. + fp := NewFakeProxier(ipt, osClient) + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: v1.ProtocolTCP, + }} + }), + ) + + epIP := "192.168.0.1" + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: epIP, + }}, + Ports: []v1.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + }}, + }} + }), + ) + + makeNamespaceMap(fp, makeTestNamespace(svcPortName.Namespace)) + + fp.syncProxyRules() + + epStr := fmt.Sprintf("%s:%d", epIP, svcPort) + + stackubeRules := ipt.GetRules(string(ChainSKPrerouting), "qrouter-123") + if len(stackubeRules) == 0 { + errorf(fmt.Sprintf("Unexpected rule for chain %v with endpoints in namespace %v", ChainSKPrerouting, svcPortName.Namespace), stackubeRules, t) + } + if !hasDNAT(stackubeRules, epStr) { + errorf(fmt.Sprintf("Chain %v lacks DNAT to %v", ChainSKPrerouting, epStr), stackubeRules, t) + } +} + +func TestMultiNamespacesService(t *testing.T) { + ns1 := "ns1" + svcIP1 := "1.2.3.4" + svcPort1 := 80 + svcPortName1 := servicePortName{ + NamespacedName: makeNSN(ns1, "svc1"), + Port: "80", + } + + ns2 := "ns2" + svcIP2 := "1.2.3.5" + svcPort2 := 8080 + svcPortName2 := servicePortName{ + NamespacedName: makeNSN(ns2, "svc1"), + Port: "8080", + } + + // Creates fake iptables. + ipt := NewFake() + // Create a fake openstack client. + osClient := openstack.NewFake() + // Injects fake network. + networkName1 := util.BuildNetworkName(ns1, ns1) + osClient.SetNetwork(networkName1, "123") + networkName2 := util.BuildNetworkName(ns2, ns2) + osClient.SetNetwork(networkName2, "456") + // Injects fake port. + osClient.SetPort("123", "network:router_interface", "123") + osClient.SetPort("456", "network:router_interface", "456") + // Creates a new fake proxier. + fp := NewFakeProxier(ipt, osClient) + + makeServiceMap(fp, + makeTestService(svcPortName1.Namespace, svcPortName1.Name, func(svc *v1.Service) { + svc.Spec.ClusterIP = svcIP1 + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName1.Port, + Port: int32(svcPort1), + Protocol: v1.ProtocolTCP, + }} + }), + makeTestService(svcPortName2.Namespace, svcPortName2.Name, func(svc *v1.Service) { + svc.Spec.ClusterIP = svcIP2 + svc.Spec.Ports = []v1.ServicePort{{ + Name: svcPortName2.Port, + Port: int32(svcPort2), + Protocol: v1.ProtocolTCP, + }} + }), + ) + + epIP1 := "192.168.0.1" + epIP2 := "192.168.1.1" + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName1.Namespace, svcPortName1.Name, func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: epIP1, + }}, + Ports: []v1.EndpointPort{{ + Name: svcPortName1.Port, + Port: int32(svcPort1), + }}, + }} + }), + makeTestEndpoints(svcPortName2.Namespace, svcPortName2.Name, func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{{ + Addresses: []v1.EndpointAddress{{ + IP: epIP2, + }}, + Ports: []v1.EndpointPort{{ + Name: svcPortName2.Port, + Port: int32(svcPort2), + }}, + }} + }), + ) + + makeNamespaceMap(fp, + makeTestNamespace(svcPortName1.Namespace), + makeTestNamespace(svcPortName2.Namespace), + ) + + fp.syncProxyRules() + + epStr1 := fmt.Sprintf("%s:%d", epIP1, svcPort1) + + ns1Rules := ipt.GetRules(string(ChainSKPrerouting), "qrouter-123") + if len(ns1Rules) == 0 { + errorf(fmt.Sprintf("Unexpected rule for chain %v with endpoints in namespace %v", ChainSKPrerouting, svcPortName1.Namespace), ns1Rules, t) + } + if !hasDNAT(ns1Rules, epStr1) { + errorf(fmt.Sprintf("Chain %v lacks DNAT to %v", ChainSKPrerouting, epStr1), ns1Rules, t) + } + + epStr2 := fmt.Sprintf("%s:%d", epIP2, svcPort2) + ns2Rules := ipt.GetRules(string(ChainSKPrerouting), "qrouter-456") + if len(ns2Rules) == 0 { + errorf(fmt.Sprintf("Unexpected rule for chain %v with endpoints in namespace %v", ChainSKPrerouting, svcPortName2.Namespace), ns2Rules, t) + } + if !hasDNAT(ns2Rules, epStr2) { + errorf(fmt.Sprintf("Chain %v lacks DNAT to %v", ChainSKPrerouting, epStr2), ns2Rules, t) + } +} + +// This is a coarse test, but it offers some modicum of confidence as the code is evolved. +func Test_endpointsToEndpointsMap(t *testing.T) { + testCases := []struct { + newEndpoints *v1.Endpoints + expected map[servicePortName][]*endpointsInfo + }{{ + // Case[0]: nothing + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) {}), + expected: map[servicePortName][]*endpointsInfo{}, + }, { + // Case[1]: no changes, unnamed port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []v1.EndpointPort{{ + Name: "", + Port: 11, + }}, + }, + } + }), + expected: map[servicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[2]: no changes, named port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []v1.EndpointPort{{ + Name: "port", + Port: 11, + }}, + }, + } + }), + expected: map[servicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "port"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[3]: new port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []v1.EndpointPort{{ + Port: 11, + }}, + }, + } + }), + expected: map[servicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[4]: remove port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) {}), + expected: map[servicePortName][]*endpointsInfo{}, + }, { + // Case[5]: new IP and port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{{ + IP: "1.1.1.1", + }, { + IP: "2.2.2.2", + }}, + Ports: []v1.EndpointPort{{ + Name: "p1", + Port: 11, + }, { + Name: "p2", + Port: 22, + }}, + }, + } + }), + expected: map[servicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + {endpoint: "2.2.2.2:11", isLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p2"): { + {endpoint: "1.1.1.1:22", isLocal: false}, + {endpoint: "2.2.2.2:22", isLocal: false}, + }, + }, + }, { + // Case[6]: remove IP and port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []v1.EndpointPort{{ + Name: "p1", + Port: 11, + }}, + }, + } + }), + expected: map[servicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[7]: rename port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []v1.EndpointPort{{ + Name: "p2", + Port: 11, + }}, + }, + } + }), + expected: map[servicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p2"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[8]: renumber port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *v1.Endpoints) { + ept.Subsets = []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []v1.EndpointPort{{ + Name: "p1", + Port: 22, + }}, + }, + } + }), + expected: map[servicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {endpoint: "1.1.1.1:22", isLocal: false}, + }, + }, + }} + + for tci, tc := range testCases { + // outputs + newEndpoints := endpointsToEndpointsMap(tc.newEndpoints, "host") + + if len(newEndpoints) != len(tc.expected) { + t.Errorf("[%d] expected %d new, got %d: %v", tci, len(tc.expected), len(newEndpoints), spew.Sdump(newEndpoints)) + } + for x := range tc.expected { + if len(newEndpoints[x]) != len(tc.expected[x]) { + t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(tc.expected[x]), x, len(newEndpoints[x])) + } else { + for i := range newEndpoints[x] { + if *(newEndpoints[x][i]) != *(tc.expected[x][i]) { + t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, tc.expected[x][i], *(newEndpoints[x][i])) + } + } + } + } + } +} diff --git a/pkg/service-controller/service_controller.go b/pkg/service-controller/service_controller.go index a364034..bfd5db8 100644 --- a/pkg/service-controller/service_controller.go +++ b/pkg/service-controller/service_controller.go @@ -332,7 +332,7 @@ func (s *ServiceController) createLoadBalancer(service *v1.Service) (*v1.LoadBal // Only support one network and network's name is same with namespace. networkName := util.BuildNetworkName(service.Namespace, service.Namespace) - network, err := s.osClient.GetNetwork(networkName) + network, err := s.osClient.GetNetworkByName(networkName) if err != nil { glog.Errorf("Get network by name %q failed: %v", networkName, err) return nil, err