diff --git a/README.md b/README.md index 0fa0c98..89a429b 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ docker-ovs-plugin ### QuickStart Instructions +The quickstart instructions describe how to start the plugin in **nat mode**. Flat mode is described in the `flat` mode section. + 1. Install the Docker experimental binary from the instructions at: [Docker Experimental](https://github.com/docker/docker/tree/master/experimental). (stop other docker instances) - Quick Experimental Install: `wget -qO- https://experimental.docker.com/ | sh` 1. Install and start Open vSwitch. @@ -25,14 +27,8 @@ docker-ovs-plugin ``` $ sudo ovs-vsctl set-manager ptcp:6640 ``` - -3. Create the `ovsbr-docker0` bridge by hand: - - ``` - $ sudo ovs-vsctl add-br ovsbr-docker0 - ``` - -4. Start Docker with the following: + +3. Start Docker with the following: ``` $sudo docker -d --default-network=ovs:ovsbr-docker0` @@ -44,48 +40,96 @@ docker-ovs-plugin # echo 'DOCKER_OPTS="--default-network=ovs:ovsbr-docker0"' >> /etc/default/docker # service docker restart ``` -5. Create the socket the plugin uses: - - ``` - $ sudo su - # mkdir -p /usr/share/docker/plugins - # touch /usr/share/docker/plugins/ovs.sock - ``` -6. Next start the plugin. A pre-compiled x86_64 binary can be downloaded from the [binaries](https://github.com/gopher-net/docker-ovs-plugin/tree/master/binaries) directory. **Note:** Running inside a container is a todo, pop it into issues if you want to help contribute that. +4. Next start the plugin. A pre-compiled x86_64 binary can be downloaded from the [binaries](https://github.com/gopher-net/docker-ovs-plugin/tree/master/binaries) directory. **Note:** Running inside a container is a todo, pop it into issues if you want to help contribute that. ``` $ wget -O ./docker-ovs-plugin https://github.com/gopher-net/docker-ovs-plugin/raw/master/binaries/docker-ovs-plugin-0.1-Linux-x86_64 $ chmod +x docker-ovs-plugin $ ./docker-ovs-plugin ``` - + + Running the binary with no options is the same as running the following. Any of those fields can be customized, just make sure your gateway is on the same network/subnet as the specified bridge subnet. + + ``` + $ ./docker-ovs-plugin --gateway=172.18.40.1 --bridge-subnet=172.18.40.0/24 -mode=nat + ``` + + If you pass a subnet but not a gateway, we currently make an assumption that the first usable address. For example, in the case of a /24 subnet the .1 on the network will be used) + For debugging or just extra logs from the sausage factory, add the debug flag `./docker-ovs-plugin -d` -6. Run some containers and verify they can ping one another with `docker run -it --rm busybox` or `docker run -it --rm ubuntu` etc, or any other docker images you prefer. Alternatively, paste a few dozen or more containers running in the background and watch the ports provision and de-provision in OVS with `docker run -itd busybox` +5. Run some containers and verify they can ping one another with `docker run -it --rm busybox` or `docker run -it --rm ubuntu` etc, or any other docker images you prefer. Alternatively, paste a few dozen or more containers running in the background and watch the ports provision and de-provision in OVS with `docker run -itd busybox` ``` - INFO[0000] OVSDB network driver initialized + INFO[0000] Plugin configuration: + container subnet: [172.18.40.0/24] + container gateway: [172.18.40.1] + bridge name: [ovsbr-docker0] + bridge mode: [nat] + mtu: [1450] + INFO[0000] OVS network driver initialized successfully INFO[0005] Dynamically allocated container IP is: [ 172.18.40.2 ] INFO[0005] Attached veth [ ovs-veth0-ac097 ] to bridge [ ovsbr-docker0 ] INFO[0009] Deleted OVS port [ ovs-veth0-ac097 ] from bridge [ ovsbr-docker0 ] ``` - **Additional Notes**: - - The argument passed to `--default-network` the plugin is identified via `ovs`. More specifically, the socket file that currently defaults to `/usr/share/docker/plugins/ovs.sock`. - - The default bridge name in the example is `ovsbr-docker0`. - - The bridge name is temporarily hardcoded. That and more will be configurable via flags. (Help us define and code those flags). +### Flat Mode + +There are two generic modes, `flat` and `nat`. The default mode is `nat` since it does not require any orchestration with the network because the address space is hidden behind iptables masquerading. + + +- flat is simply an OVS bridge with the container link attached to it. An example would be a Docker host is plugged into a data center port that has a subnet of `192.168.1.0/24`. You would start the plugin like so: + +``` +$ docker-ovs-plugin --gateway=192.168.1.1 --bridge-subnet=192.168.1.0/24 -mode=flat +``` + +- Containers now start attached to an OVS bridge. It could be tagged or untagged but either way it is isolated and unable to communicate to anything outside of its bridge domain. In this case, you either add VXLAN tunnels to other bridges of the same bridge domain or add an `eth` interface to the bridge to allow access to the underlying network when traffic leaves the Docker host. To do so, you simply add the `eth` interface to the ovs bridge. Neither the bridge nor the eth interface need to have an IP address since traffic from the container is strictly L2. **Warning** if you are remoted into the physical host make sure you are not using an ethernet interface to attach to the bridge that is also your management interface since the eth interface no longer uses the IP address it had. The IP would need to be migrated to ovsbr-docker0 in this case. Allowing underlying network access to an OVS bridge can be done like so: + +``` +ovs-vsctl add-port ovsbr-docker0 eth2 + +``` + +Add an address to ovsbr-docker0 if you want an L3 interface on the L2 domain for the Docker host if you would like one for troubleshooting etc but it isn't required since flat mode cares only about MAC addresses and VLAN IDs like any other L2 domain would. + +- Example of OVS with an ethernet interface bound to it for external access to the container sitting on the same bridge. NAT mode doesn't need the eth interface because IPTables is doing NAT/PAAT instead of bridging all the way through. + + +``` +$ ovs-vsctl show +e0de2079-66f0-4279-a1c8-46ba0672426e + Manager "ptcp:6640" + is_connected: true + Bridge "ovsbr-docker0" + Port "ovsbr-docker0" + Interface "ovsbr-docker0" + type: internal + Port "ovs-veth0-d33a9" + Interface "ovs-veth0-d33a9" + Port "eth2" + Interface "eth2" + ovs_version: "2.3.1" +``` + + +### Additional Notes: + + - The argument passed to `--default-network` the plugin is identified via `ovs`. More specifically, the socket file that currently defaults to `/run/docker/plugins/ovs.sock`. + - The default bridge name in the example is `ovsbr-docker0`. + - The bridge name is temporarily hardcoded. That and more will be configurable via flags. (Help us define and code those flags). - Add other flags as desired such as `--dns=8.8.8.8` for DNS etc. - To view the Open vSwitch configuration, use `ovs-vsctl show`. - - To view the OVSDB tables, run `ovsdb-client dump`. All of the mentioned OVS utils are part of the standard binary installations with very well documented [man pages](http://openvswitch.org/support/dist-docs/). - - The containers are brought up on a flat bridge. This means there is no NATing occurring. A layer 2 adjacency such as a VLAN or overlay tunnel is required for multi-host communications. If the traffic needs to be routed an external process to act as a gateway (on the TODO list so dig in if interested in multi-host or overlays). - - Download a quick video demo [here](https://dl.dropboxusercontent.com/u/51927367/Docker-OVS-Plugin.mp4). - + - To view the OVSDB tables, run `ovsdb-client dump`. All of the mentioned OVS utils are part of the standard binary installations with very well documented [man pages](http://openvswitch.org/support/dist-docs/). + - The containers are brought up on a flat bridge. This means there is no NATing occurring. A layer 2 adjacency such as a VLAN or overlay tunnel is required for multi-host communications. If the traffic needs to be routed an external process to act as a gateway (on the TODO list so dig in if interested in multi-host or overlays). + - Download a quick video demo [here](https://dl.dropboxusercontent.com/u/51927367/Docker-OVS-Plugin.mp4). + ### Hacking and Contributing Yes!! Please see issues for todos or add todos into [issues](https://github.com/gopher-net/docker-ovs-plugin/issues)! Only rule here is no jerks. -Since this plugin uses netlink for L3 IP assigments, a Linux host that can build [vishvananda/netlink](https://github.com/vishvananda/netlink) library is required. +Since this plugin uses netlink for L3 IP assignments, a Linux host that can build [vishvananda/netlink](https://github.com/vishvananda/netlink) library is required. 1. Install [Go](https://golang.org/doc/install). OVS as listed above and a kernel >= 3.19. @@ -94,9 +138,11 @@ Since this plugin uses netlink for L3 IP assigments, a Linux host that can build ``` git clone https://github.com/gopher-net/docker-ovs-plugin.git cd docker-ovs-plugin/plugin - # Get the Go dependdencies + # Get the Go dependencies go get ./... go run main.go + # or using explicit configuration flags: + go run main.go -d --gateway=172.18.40.1 --bridge-subnet=172.18.40.0/24 -mode=nat ``` 3. The rest is the same as the Quickstart Section. diff --git a/Vagrantfile b/Vagrantfile index 5d4e9ec..ec97922 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,7 +15,7 @@ ovs-vsctl set-manager ptcp:6640 echo DOCKER_OPTS=\\"--default-network=ovs:ovsbr-docker0\\" >> /etc/default/docker service docker restart mkdir -p /usr/share/docker/plugins -touch /usr/share/docker/plugins/ovs.sock +touch /run/docker/plugins/ovs.sock wget -O /home/vagrant/docker-ovs-plugin https://github.com/gopher-net/docker-ovs-plugin/raw/master/binaries/docker-ovs-plugin-0.1-Linux-x86_64 chmod +x /home/vagrant/docker-ovs-plugin SCRIPT diff --git a/binaries/docker-ovs-plugin-0.1-Linux-x86_64 b/binaries/docker-ovs-plugin-0.1-Linux-x86_64 old mode 100755 new mode 100644 diff --git a/docker-compose.yml b/docker-compose.yml index ee9cae6..2739a32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ plugin: build: . volumes: - - /usr/share/docker/plugins/ovs.sock:/usr/share/docker/plugins/ovs.sock + - /run/docker/plugins/ovs.sock:/run/docker/plugins/ovs.sock - /var/run/docker.sock:/var/run/docker.sock net: host privileged: true diff --git a/install.sh b/install.sh index 1286bfd..f530d31 100644 --- a/install.sh +++ b/install.sh @@ -1,4 +1,4 @@ #!/bin/sh -touch /usr/share/docker/plugins/ovs.sock +touch /run/docker/plugins/ovs.sock docker-compose up -d diff --git a/plugin/main.go b/plugin/main.go index d63d80b..aeb697e 100644 --- a/plugin/main.go +++ b/plugin/main.go @@ -1,21 +1,25 @@ package main import ( + "fmt" "os" - "path/filepath" log "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" "github.com/gopher-net/docker-ovs-plugin/plugin/ovs" ) -const version = "0.1" +const ( + version = "0.1" + ovsSocket = "ovs.sock" + pluginPath = "/run/docker/plugins/" +) func main() { var flagSocket = cli.StringFlag{ Name: "socket, s", - Value: "/usr/share/docker/plugins/ovs.sock", + Value: ovsSocket, Usage: "listening unix socket", } var flagDebug = cli.BoolFlag{ @@ -29,16 +33,17 @@ func main() { app.Flags = []cli.Flag{ flagDebug, flagSocket, - ovs.FlagBridgeName, - ovs.FlagBridgeIP, ovs.FlagBridgeSubnet, + ovs.FlagIpVlanMode, + ovs.FlagGateway, + ovs.FlagMtu, } app.Action = Run - app.Before = cliInit + app.Before = initEnv app.Run(os.Args) } -func cliInit(ctx *cli.Context) error { +func initEnv(ctx *cli.Context) error { socketFile := ctx.String("socket") // Default loglevel is Info if ctx.Bool("debug") { @@ -47,21 +52,7 @@ func cliInit(ctx *cli.Context) error { log.SetLevel(log.InfoLevel) } log.SetOutput(os.Stderr) - // Verify the path to the plugin socket oath and filename were passed - sockDir, fileHandle := filepath.Split(socketFile) - if fileHandle == "" { - log.Fatalf("Socket file path and name are required. Ex. /usr/share/docker/plugins/.sock") - } - // Make the plugin filepath and parent dir if it does not already exist - if err := os.MkdirAll(sockDir, 0755); err != nil && !os.IsExist(err) { - log.Warnf("Could not create net plugin path directory: [ %s ]", err) - } - // If the plugin socket file already exists, remove it. - if _, err := os.Stat(socketFile); err == nil { - log.Debugf("socket file [ %s ] already exists, deleting..", socketFile) - removeSock(socketFile) - } - log.Debugf("Plugin socket path is [ %s ] with a file handle [ %s ]", sockDir, fileHandle) + initSock(socketFile) return nil } @@ -69,18 +60,37 @@ func cliInit(ctx *cli.Context) error { func Run(ctx *cli.Context) { var d ovs.Driver var err error - if d, err = ovs.New(version); err != nil { + if d, err = ovs.New(version, ctx); err != nil { log.Fatalf("unable to create driver: %s", err) } - log.Info("OVSDB network driver initialized") - if err := d.Listen(ctx.String("socket")); err != nil { + log.Info("OVS network driver initialized successfully") + + // concatenate the absolute path to the spec file handle + absSocket := fmt.Sprint(pluginPath, ctx.String("socket")) + if err := d.Listen(absSocket); err != nil { log.Fatal(err) } } -func removeSock(sockFile string) { - err := os.Remove(sockFile) +// removeSock if an old filehandle exists remove it +func removeSock(absFile string) { + err := os.RemoveAll(absFile) if err != nil { - log.Fatalf("unable to remove old socket file [ %s ] due to: %s", sockFile, err) + log.Fatalf("Unable to remove the old socket file [ %s ] due to: %s", absFile, err) + } +} + +// initSock create the plugin filepath if it does not already exist +func initSock(socketFile string) { + if err := os.MkdirAll(pluginPath, 0755); err != nil && !os.IsExist(err) { + log.Warnf("Could not create net plugin path directory: [ %s ]", err) + } + // concatenate the absolute path to the spec file handle + absFile := fmt.Sprint(pluginPath, socketFile) + // If the plugin socket file already exists, remove it. + if _, err := os.Stat(absFile); err == nil { + log.Debugf("socket file [ %s ] already exists, unlinking the old file handle..", absFile) + removeSock(absFile) } + log.Debugf("The plugin absolute path and handle is [ %s ]", absFile) } diff --git a/plugin/ovs/cli.go b/plugin/ovs/cli.go index 2da0bbc..ba98658 100644 --- a/plugin/ovs/cli.go +++ b/plugin/ovs/cli.go @@ -4,17 +4,19 @@ import "github.com/codegangsta/cli" // Exported variables var ( - // TODO: Values need to be bound to driver. Need to modify the Driver iface. Added brOpts if we want to pass that to Listen(string) - FlagBridgeName = cli.StringFlag{Name: "bridge-name", Value: bridgeName, Usage: "name of the OVS bridge to add containers. If it doees not exist, it will be created. default: --bridge-name=ovsbr-docker0"} - FlagBridgeIP = cli.StringFlag{Name: "bridge-net", Value: bridgeIfaceNet, Usage: "IP and netmask of the bridge. default: --bridge-ip=172.18.40.1/24"} - FlagBridgeSubnet = cli.StringFlag{Name: "bridge-subnet", Value: bridgeSubnet, Usage: "subnet for the containers on the bridge to use (currently IPv4 support). default: --bridge-subnet=172.18.40.0/24"} + FlagIpVlanMode = cli.StringFlag{Name: "mode", Value: ovsDriverMode, Usage: "name of the OVS driver mode [nat|flat]. (default: l2)"} + FlagBridgeSubnet = cli.StringFlag{Name: "bridge-subnet", Value: bridgeSubnet, Usage: "(required for flat L2 mode) subnet for the containers on the bridge to use. default only applies to NAT mode: --bridge-subnet=172.18.40.0/24"} + FlagMtu = cli.IntFlag{Name: "mtu", Value: defaultMTU, Usage: "MTU of the container interface (default: 1440 Note: greater then 1500 unsupported atm)"} + FlagGateway = cli.StringFlag{Name: "gateway", Value: gatewayIP, Usage: "(required for flat L2 mode) IP of the default gateway (default NAT mode: 172.18.40.1)"} + // Bridge name currently needs to match the docker -run bridge name. Leaving this unmodifiable until that is sorted + FlagBridgeName = cli.StringFlag{Name: "bridge-name", Value: bridgeName, Usage: "name of the OVS bridge to add containers. (default name: ovsbr-docker0"} ) // Unexported variables var ( - // TODO: Temp hardcodes, bind to CLI flags and/or dnet-ctl for bridge properties. - bridgeName = "ovsbr-docker0" // temp until binding via flags - bridgeSubnet = "172.18.40.0/24" // temp until binding via flags - bridgeIfaceNet = "172.18.40.1/24" // temp until binding via flags - gatewayIP = "172.18.40.1" // Bridge vs. GW IPs + bridgeName = "ovsbr-docker0" // TODO: currently immutable + bridgeSubnet = "172.18.40.0/24" // NAT mode can use this addr. Flat (L2) mode requires an IPNet that will overwrite this val. + gatewayIP = "" // NAT mode will use the first usable address of the bridgeSubnet."172.18.40.0/24" would use "172.18.40.1" as a gateway. Flat L2 mode requires an external gateway for L3 routing + ovsDriverMode = "nat" // Default mode is NAT. + defaultMTU = 1450 ) diff --git a/plugin/ovs/driver.go b/plugin/ovs/driver.go index 46d1814..3227702 100644 --- a/plugin/ovs/driver.go +++ b/plugin/ovs/driver.go @@ -6,11 +6,10 @@ import ( "io" "net" "net/http" - "strconv" + "strings" log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/ipallocator" - "github.com/docker/libnetwork/types" "github.com/gorilla/mux" "github.com/vishvananda/netlink" ) @@ -26,11 +25,13 @@ type Driver interface { Listen(string) error } -// Struct for binding bridge options CLI flags -type bridgeOpts struct { - brName string - brSubnet net.IPNet - brIP net.IPNet +// Struct for binding plugin specific configurations (cli.go for details). +type pluginConfig struct { + mtu int + bridgeName string + mode string + brSubnet *net.IPNet + gatewayIP net.IP } type driver struct { @@ -42,6 +43,7 @@ type driver struct { cidr *net.IPNet nameserver string OvsdbNotifier + pluginConfig } func (driver *driver) Listen(socket string) error { @@ -141,14 +143,7 @@ func (driver *driver) createNetwork(w http.ResponseWriter, r *http.Request) { return } driver.network = create.NetworkID - - _, ipNet, err := net.ParseCIDR(bridgeSubnet) - if err != nil { - log.Warnf("Error parsing cidr from the default subnet: %s", err) - } - cidr := ipNet - driver.cidr = cidr - driver.ipAllocator.RequestIP(cidr, nil) + driver.ipAllocator.RequestIP(driver.pluginConfig.brSubnet, nil) emptyResponse(w) } @@ -170,8 +165,6 @@ func (driver *driver) deleteNetwork(w http.ResponseWriter, r *http.Request) { return } driver.network = "" - // TODO: needs work for multi-bridge. NetworkID needs to be bound to a bridgeName - // TODO: needs testing err := driver.deleteBridge(bridgeName) if err != nil { log.Errorf("Deleting bridge:[ %s ] and network:[ %s ] failed: %s", bridgeName, bridgeSubnet, err) @@ -205,19 +198,24 @@ func (driver *driver) createEndpoint(w http.ResponseWriter, r *http.Request) { sendError(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) return } - // If the bridge has been created, a port with the same name should exist + + // If the bridge has been created, an OVSDB port table row should exist exists, err := driver.ovsdber.portExists(bridgeName) if err != nil { - log.Debugf("OVS bridge already exists: %s", err) - driver.verifyBridge() + log.Debugf("Error querying the ovsdb cache: %s", err) } + // If the bridge does not exist create and assign an IP if !exists { err := driver.setupBridge() if err != nil { log.Errorf("unable to setup the OVS bridge [ %s ]: %s ", bridgeName, err) } + } else { + log.Debugf("OVS bridge [ %s ] already exists, verifying its configuration.", bridgeName) + driver.verifyBridgeIp() } + // Bring the bridge up err = driver.interfaceUP(bridgeName) if err != nil { @@ -232,17 +230,19 @@ func (driver *driver) createEndpoint(w http.ResponseWriter, r *http.Request) { } // Request an IP address from libnetwork based on the cidr scope // TODO: Add a user defined static ip addr option - allocatedIP, err := driver.ipAllocator.RequestIP(driver.cidr, nil) + allocatedIP, err := driver.ipAllocator.RequestIP(driver.pluginConfig.brSubnet, nil) if err != nil || allocatedIP == nil { log.Errorf("Unable to obtain an IP address from libnetwork ipam: %s", err) errorResponsef(w, "%s", err) return } + // generate a mac address for the pending container mac := makeMac(allocatedIP) // Have to convert container IP to a string ip/mask format - _, containerMask := driver.cidr.Mask.Size() - containerAddress := allocatedIP.String() + "/" + strconv.Itoa(containerMask) + bridgeMask := strings.Split(driver.pluginConfig.brSubnet.String(), "/") + containerAddress := allocatedIP.String() + "/" + bridgeMask[1] + log.Infof("Dynamically allocated container IP is: [ %s ]", allocatedIP.String()) respIface := &iface{ @@ -253,6 +253,13 @@ func (driver *driver) createEndpoint(w http.ResponseWriter, r *http.Request) { Interfaces: []*iface{respIface}, } objectResponse(w, resp) + + if driver.pluginConfig.mode == modeNAT { + err := driver.natOut() + if err != nil { + log.Errorf("Error setting NAT mode iptable rules for OVS bridge [ %s ]: %s ", driver.pluginConfig.mode, err) + } + } log.Debugf("Create endpoint %s %+v", endID, resp) } @@ -351,14 +358,6 @@ func (driver *driver) joinEndpoint(w http.ResponseWriter, r *http.Request) { if err != nil { log.Warnf("Error enabling Veth local iface: [ %v ]", local) } - /* ------- Attaching Veth pair via ovsdb no worky -------- */ - // Attach half of the veth pair as an OVS port to the OVS bridge - // err = driver.ovsdber.addOvsVethPort(local.Name, bridgeName, 0) - // if err != nil { - // log.Debugf("error attaching veth [ %s ] to bridge [ %s ]", local.Name, bridgeName) - // errorResponsef(w, "%s", err) - // } - err = driver.addPortExec(bridgeName, local.Name) if err != nil { log.Errorf("error attaching veth [ %s ] to bridge [ %s ]", local.Name, bridgeName) @@ -374,16 +373,8 @@ func (driver *driver) joinEndpoint(w http.ResponseWriter, r *http.Request) { } res := &joinResponse{ InterfaceNames: []*iface{ifname}, - Gateway: gatewayIP, - } - // Add Connected Route - routeToDNS := &staticRoute{ - Destination: bridgeSubnet, - RouteType: types.CONNECTED, - NextHop: "", - InterfaceID: 0, + Gateway: driver.pluginConfig.gatewayIP.String(), } - res.StaticRoutes = []*staticRoute{routeToDNS} objectResponse(w, res) log.Debugf("Join endpoint %s:%s to %s", j.NetworkID, j.EndpointID, j.SandboxKey) diff --git a/plugin/ovs/ovs_bridge.go b/plugin/ovs/ovs_bridge.go index 685cc1f..1012d64 100644 --- a/plugin/ovs/ovs_bridge.go +++ b/plugin/ovs/ovs_bridge.go @@ -3,9 +3,10 @@ package ovs import ( "errors" "fmt" - "net" + "strings" log "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/iptables" "github.com/socketplane/libovsdb" ) @@ -15,19 +16,19 @@ func (driver *driver) setupBridge() error { log.Errorf("error creating ovs bridge [ %s ] : [ %s ]", bridgeName, err) return err } - // Set bridge IP. - brAddr, _, err := net.ParseCIDR(bridgeIfaceNet) - if err != nil { - log.Warnf("Error parsing cidr from the default subnet: %s", err) - } - // Set the L3 IP addr on the bridge netlink iface - if bridgeIfaceNet != "" { - err := driver.setInterfaceIP(bridgeName, bridgeIfaceNet) - // if the bridge IP and it is the gateway log an error. TODO: Check if the brIP exists already - if err != nil && gatewayIP == brAddr.String() { - log.Debugf("Error assigning address : [ %s ] on bridge: [ %s ] with an error of: %s", bridgeSubnet, bridgeName, err) + + // Set the L3 addr on the bridge's netlink iface if in NAT since it needs to masq + if driver.pluginConfig.mode == modeNAT { + // create a string representation of the bridge address in cidr form (ip/mask) + bridgeMask := strings.Split(driver.pluginConfig.brSubnet.String(), "/") + bridgeCidr := fmt.Sprintf("%s/%s", driver.pluginConfig.gatewayIP, bridgeMask[1]) + // Set bridge IP. + err := driver.setInterfaceIP(bridgeName, bridgeCidr) + if err != nil { + log.Debugf("Error assigning address:[ %s ] on bridge:[ %s ] with an error of: %s", bridgeCidr, bridgeName, err) } } + // Verify there is an IP on the netlink iface. If it is the gateway it is a problem. brNet, err := getIfaceAddr(bridgeName) if err != nil { @@ -39,27 +40,29 @@ func (driver *driver) setupBridge() error { } // verifyBridge is if the bridge already existed and ensures it has a netlink L3 IP -func (driver *driver) verifyBridge() error { +func (driver *driver) verifyBridgeIp() error { // Verify there is an IP on the bridge brNet, err := getIfaceAddr(bridgeName) - if err == nil { + if brNet != nil { log.Debugf("IP address [ %s ] found on bridge: [ %s ]", brNet, bridgeName) return nil } - // Parse the bridge IP. - brAddr, _, err := net.ParseCIDR(bridgeSubnet) - if err != nil { - log.Errorf("Error parsing cidr from the default subnet: %s", err) - return err - } - if bridgeIfaceNet != "" { - err := driver.setInterfaceIP(bridgeName, bridgeIfaceNet) - // if the there is an error setting the br IP and it is the gateway return an err - if err != nil && gatewayIP == brAddr.String() { - log.Warnf("Error assigning address : [ %s ] on bridge: [ %s ] with an error of: %s", bridgeSubnet, bridgeName, err) + + // Set the L3 addr on the bridge's netlink iface if in NAT since it needs to masq + if driver.pluginConfig.mode == modeNAT { + // create a string representation of the bridge address in cidr form (ip/mask) + bridgeMask := strings.Split(driver.pluginConfig.brSubnet.String(), "/") + bridgeCidr := fmt.Sprintf("%s/%s", driver.pluginConfig.gatewayIP, bridgeMask[1]) + + // Set bridge IP. + log.Debugf("Assigning IP address [ %s ] to bridge: [ %s ]", brNet, bridgeName) + err := driver.setInterfaceIP(bridgeName, bridgeCidr) + if err != nil { + log.Debugf("Error assigning address:[ %s ] on bridge:[ %s ] with an error of: %s", bridgeCidr, bridgeName, err) } } - return nil + + return err } func (ovsdber *ovsdber) createBridgeIface(name string) error { @@ -212,3 +215,26 @@ func (driver *driver) deleteBridge(bridgeName string) error { log.Debugf("OVSDB delete bridge transaction succesful") return nil } + +// todo: reconcile with what libnetwork does and port mappings +func (driver *driver) natOut() error { + masquerade := []string{ + "POSTROUTING", "-t", "nat", + "-s", driver.pluginConfig.brSubnet.String(), + "-j", "MASQUERADE", + } + if _, err := iptables.Raw( + append([]string{"-C"}, masquerade...)..., + ); err != nil { + incl := append([]string{"-I"}, masquerade...) + if output, err := iptables.Raw(incl...); err != nil { + return err + } else if len(output) > 0 { + return &iptables.ChainError{ + Chain: "POSTROUTING", + Output: output, + } + } + } + return nil +} diff --git a/plugin/ovs/ovsdb.go b/plugin/ovs/ovsdb.go index ca88b9d..5d8e172 100644 --- a/plugin/ovs/ovsdb.go +++ b/plugin/ovs/ovsdb.go @@ -3,10 +3,12 @@ package ovs import ( "errors" "fmt" + "net" "reflect" "time" log "github.com/Sirupsen/logrus" + "github.com/codegangsta/cli" "github.com/docker/libnetwork/ipallocator" "github.com/samalba/dockerclient" "github.com/socketplane/libovsdb" @@ -17,6 +19,9 @@ const ( ovsdbPort = 6640 contextKey = "container_id" contextValue = "container_data" + minMTU = 68 + modeFlat = "flat" // L2 driver mode + modeNAT = "nat" // NAT hides backend bridge network ) var ( @@ -70,15 +75,90 @@ func (ovsdber *ovsdber) initDBCache() { } } -func New(version string) (Driver, error) { +func New(version string, ctx *cli.Context) (Driver, error) { docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil) if err != nil { return nil, fmt.Errorf("could not connect to docker: %s", err) } + + // initiate the ovsdb manager port binding ovsdb, err := libovsdb.Connect(localhost, ovsdbPort) if err != nil { return nil, fmt.Errorf("could not connect to openvswitch on port [ %d ]: %s", ovsdbPort, err) } + + // bind user defined flags to the plugin config + if ctx.String("bridge-name") != "" { + bridgeName = ctx.String("bridge-name") + } + + // lower bound of v4 MTU is 68-bytes per rfc791 + if ctx.Int("mtu") >= minMTU { + defaultMTU = ctx.Int("mtu") + } else { + log.Fatalf("The MTU value passed [ %d ] must be greater then [ %d ] bytes per rfc791", ctx.Int("mtu"), minMTU) + } + + // Parse the container subnet + containerGW, containerCidr, err := net.ParseCIDR(ctx.String("bridge-subnet")) + if err != nil { + log.Fatalf("Error parsing cidr from the subnet flag provided [ %s ]: %s", FlagBridgeSubnet, err) + } + + // Update the cli.go global var with the network if user provided + bridgeSubnet = containerCidr.String() + + switch ctx.String("mode") { + /* [ flat ] mode */ + //Flat mode requires a gateway IP address is used just like any other + //normal L2 domain. If no gateway is specified, we attempt to guess using + //the first usable IP on the container subnet from the CLI argument. + //Example "192.168.1.0/24" we guess at a gatway of "192.168.1.1". + //Flat mode requires a bridge-subnet flag with a subnet from your existing network + case modeFlat: + ovsDriverMode = modeFlat + if ctx.String("gateway") != "" { + // bind the container gateway to the IP passed from the CLI + cliGateway := net.ParseIP(ctx.String("gateway")) + if cliGateway == nil { + log.Fatalf("The IP passed with the [ gateway ] flag [ %s ] was not a valid address: %s", FlagGateway.Value, err) + } + containerGW = cliGateway + } else { + // if no gateway was passed, guess the first valid address on the container subnet + containerGW = ipIncrement(containerGW) + } + /* [ nat ] mode */ + //If creating a private network that will be NATed on the OVS bridge via IPTables + //it is not required to pass a subnet since in a single host scenario it is hidden + //from the network once it is masqueraded via IP tables. + case modeNAT, "": + ovsDriverMode = modeNAT + if ctx.String("gateway") != "" { + // bind the container gateway to the IP passed from the CLI + cliGateway := net.ParseIP(ctx.String("gateway")) + if cliGateway == nil { + log.Fatalf("The IP passed with the [ gateway ] flag [ %s ] was not a valid address: %s", FlagGateway.Value, err) + } + containerGW = cliGateway + } else { + // if no gateway was passed, guess the first valid address on the container subnet + containerGW = ipIncrement(containerGW) + } + default: + log.Fatalf("Invalid ovs mode supplied [ %s ]. The plugin currently supports two modes: [ %s ] or [ %s ]", ctx.String("mode"), modeFlat, modeNAT) + } + + pluginOpts := &pluginConfig{ + mtu: defaultMTU, + bridgeName: bridgeName, + mode: ovsDriverMode, + brSubnet: containerCidr, + gatewayIP: containerGW, + } + // Leaving as info for now. Change to debug eventually + log.Infof("Plugin configuration: \n %s", pluginOpts) + ipAllocator := ipallocator.New() d := &driver{ dockerer: dockerer{ @@ -87,8 +167,9 @@ func New(version string) (Driver, error) { ovsdber: ovsdber{ ovsdb: ovsdb, }, - ipAllocator: ipAllocator, - version: version, + ipAllocator: ipAllocator, + pluginConfig: *pluginOpts, + version: version, } // Initialize ovsdb cache at rpc connection setup d.ovsdber.initDBCache() @@ -188,3 +269,13 @@ func populateCache(updates libovsdb.TableUpdates) { } } } + +// return string representation of pluginConfig for debugging +func (d *pluginConfig) String() string { + str := fmt.Sprintf(" container subnet: [%s]\n", d.brSubnet.String()) + str = str + fmt.Sprintf(" container gateway: [%s]\n", d.gatewayIP.String()) + str = str + fmt.Sprintf(" bridge name: [%s]\n", d.bridgeName) + str = str + fmt.Sprintf(" bridge mode: [%s]\n", d.mode) + str = str + fmt.Sprintf(" mtu: [%d]", d.mtu) + return str +} diff --git a/plugin/ovs/utils.go b/plugin/ovs/utils.go index 93a42a4..1ba8d3a 100644 --- a/plugin/ovs/utils.go +++ b/plugin/ovs/utils.go @@ -41,8 +41,8 @@ func getIfaceAddr(name string) (*net.IPNet, error) { func (driver *driver) setInterfaceIP(name string, rawIP string) error { iface, err := netlink.LinkByName(name) if err != nil { - log.Debugf("error retrieving new OVS bridge link [ %s ] likely sync between ovs and netlink, retrying in 1 second..", bridgeName) - time.Sleep(1 * time.Second) + log.Debugf("error retrieving new OVS bridge link [ %s ] likely a race issue between ovs and netlink, retrying in 1 second..", bridgeName) + time.Sleep(2 * time.Second) iface, err = netlink.LinkByName(name) if err != nil { log.Errorf("Error retrieving the new OVS bridge from netlink: %s", err) @@ -57,7 +57,7 @@ func (driver *driver) setInterfaceIP(name string, rawIP string) error { return netlink.AddrAdd(iface, addr) } -// Increment a subnet +// Increment an IP in a subnet func ipIncrement(networkAddr net.IP) net.IP { for i := 15; i >= 0; i-- { b := networkAddr[i]