Skip to content
This repository has been archived by the owner on Jul 5, 2022. It is now read-only.

Commit

Permalink
Use CNM to replace-create CNM fixed-ip container (#46)
Browse files Browse the repository at this point in the history 
* use CNM to replace-create CNM fixed-ip container

* fetch CNM reserved IP by ippool

* update docker-cni version
  • Loading branch information
@jschwinger233
jschwinger233 authored Mar 8, 2022
1 parent 12664e8 commit 624d490
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 23 deletions.
5 changes: 4 additions & 1 deletion cni/handler/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@ func (h *BarrelHandler) HandleCNIConfig(config []byte) (newConfig []byte, err er
cniArgs := os.Getenv("CNI_ARGS")
ippool := ""
for _, args := range strings.Split(cniArgs, ";") {
if args == "" {
continue
}
parts := strings.Split(args, "=")
if len(parts) != 2 {
return nil, fmt.Errorf("invalid CNI_ARGS: %s", args)
return nil, fmt.Errorf("invalid CNI_ARGS: '%s'", cniArgs)
}
if parts[0] == "IPPOOL" {
ippool = parts[1]
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ require (
github.com/projectcalico/go-yaml-wrapper v0.0.0-20191112210931-090425220c54 // indirect
github.com/projectcalico/libcalico-go v3.9.0-0.dev+incompatible
github.com/projectcalico/libnetwork-plugin v1.1.3
github.com/projecteru2/docker-cni v0.0.1-rc.4
github.com/projecteru2/docker-cni v0.0.1-rc.5
github.com/prometheus/client_golang v1.7.1 // indirect
github.com/prometheus/procfs v0.2.0 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
Expand All @@ -67,7 +67,7 @@ require (
go.uber.org/automaxprocs v1.3.0
go.uber.org/zap v1.16.0 // indirect
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 // indirect
google.golang.org/appengine v1.6.5 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,8 @@ github.com/projecteru2/docker-cni v0.0.1-rc.3 h1:tnNh7IOg8zK0OAIawKAmGnpRNWVhffS
github.com/projecteru2/docker-cni v0.0.1-rc.3/go.mod h1:4sXRZ9D5SLC7SMy1mjKadqAx0RUw0v3vAgMvlF7XfIs=
github.com/projecteru2/docker-cni v0.0.1-rc.4 h1:Rm5J+DV0uXkshUeOL4+9J8cQJss2L3PjPlCje1Xvprg=
github.com/projecteru2/docker-cni v0.0.1-rc.4/go.mod h1:4sXRZ9D5SLC7SMy1mjKadqAx0RUw0v3vAgMvlF7XfIs=
github.com/projecteru2/docker-cni v0.0.1-rc.5 h1:YrGCeqqUIiELGdfKK40yrnlwQ4Xe+TJI5FAOC+juSjE=
github.com/projecteru2/docker-cni v0.0.1-rc.5/go.mod h1:4sXRZ9D5SLC7SMy1mjKadqAx0RUw0v3vAgMvlF7XfIs=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
Expand Down Expand Up @@ -520,6 +522,8 @@ golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
Expand Down
60 changes: 47 additions & 13 deletions proxy/docker/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"net/http"
"regexp"
"strings"
"time"

"github.com/projecteru2/barrel/cni/subhandler"
barrelHttp "github.com/projecteru2/barrel/http"
Expand Down Expand Up @@ -297,11 +298,12 @@ func (handler containerCreateHandler) writeServerResponse(
}

// steps:
// 0. condition: enable cni && custom network
// 0. condition: enable cni && custom network && specific IP not allocated by CNM
// 1. force --net none
// 2. --label cni => --runtime barrel-cni
// 3. --label fixed-ip=1 => --env fixed-ip=1
// 4. --label ipv4=x => --env ipv4=x
// 2. force --runtime barrel-cni
// 3. if Labels[fixed-ip]=1 then --env fixed-ip=1
// 4. if NetworkingConfig.EndpointsConfig.IPAMConfig.IPv4Address=x then --env IPV4=x
// 5. if HostConfig.NetworkMode=x then --env IPPOOL=x
func (handler containerCreateHandler) adaptRequestForCNI(body utils.Object) (err error) {
var (
hostConfig utils.Object
Expand Down Expand Up @@ -339,22 +341,35 @@ func (handler containerCreateHandler) adaptRequestForCNI(body utils.Object) (err
}

if !handler.cniBase.Enabled() {
logger.Infof("cni diabled, skip cni mode")
return
}

todo := []func(){}
defer func() {
for _, f := range todo {
f()
}
}()

if iNetworkMode, ok := hostConfig.Get("NetworkMode"); !ok || iNetworkMode.Null() {
return
} else if networkMode, ok = iNetworkMode.StringValue(); !ok {
return errors.Errorf("parse NetworkMode error, networkMode=%s", iNetworkMode.String())
}
if !isCustomNetwork(networkMode) {
logger.Infof("not custom network, skip cni mode")
return
}
env.Add(utils.NewStringNode("IPPOOL=" + networkMode))

logger.Info("cni mode enabled, set network none, set runtime barrel-cni")
hostConfig.Set("Runtime", utils.NewStringNode("barrel-cni"))
hostConfig.Set("NetworkMode", utils.NewStringNode("none"))
todo = append(todo,
func() {
logger.Infof("cni mode enabled, set network none, add env IPPOOL=%s, set runtime barrel-cni", networkMode)
env.Add(utils.NewStringNode("IPPOOL=" + networkMode))
hostConfig.Set("Runtime", utils.NewStringNode("barrel-cni"))
hostConfig.Set("NetworkMode", utils.NewStringNode("none"))
},
)
networkConfig, err := ensureObjectMember(body, "NetworkingConfig")
if err != nil {
return
Expand All @@ -372,16 +387,35 @@ func (handler containerCreateHandler) adaptRequestForCNI(body utils.Object) (err
specificIP, _ = ipv4Address.StringValue()
}
}
networkConfig.Set("EndpointsConfig", utils.NewObjectNode().Any())
todo = append(todo, func() {
logger.Info("cni mode enabled, empty EndpointConfig")
networkConfig.Set("EndpointsConfig", utils.NewObjectNode().Any())
})

if fixedIPLabel, ok := labels.Get(FixedIPLabel); ok && flagEnabled(fixedIPLabel) {
logger.Info("cni fixed-ip mode detected, set fixed-ip env")
env.Add(utils.NewStringNode(FixedIPLabel + "=1"))
todo = append(todo, func() {
logger.Info("cni fixed-ip mode detected, set fixed-ip env")
env.Add(utils.NewStringNode(FixedIPLabel + "=1"))
})
}

if specificIP != "" {
logger.Info("cni specific-ip mode detected, set ipv4 env")
env.Add(utils.NewStringNode("IPV4=" + specificIP))
todo = append(todo, func() {
logger.Infof("cni specific-ip mode detected, set ipv4 env IPV4=%s", specificIP)
env.Add(utils.NewStringNode("IPV4=" + specificIP))
})

ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if _, err := handler.vess.FixedIPAllocator().GetFixedIP(ctx, types.IP{Address: specificIP, PoolID: networkMode}, nil); err != nil {
if err == types.ErrFixedIPNotAllocated {
return nil
}
return err
}
logger.Infof("specific IP allocated by CNM, skip cni mode: %s", specificIP)
todo = []func(){}
return nil
}

return nil
Expand Down
15 changes: 8 additions & 7 deletions vessel/fixedippool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type FixedIPPoolManager interface {
AssignFixedIP(context.Context, types.IP) error
UnassignFixedIP(context.Context, types.IP) error
UnallocFixedIP(context.Context, types.IP, bool) error
GetFixedIP(context.Context, types.IP, func(context.Context, types.IP, *codecs.IPInfoCodec) error) (*codecs.IPInfoCodec, error)
}

// FixedIPPool .
Expand Down Expand Up @@ -64,7 +65,7 @@ func (pool fixedIPPool) AssignFixedIP(ctx context.Context, ip types.IP) error {
err error
)

if ipInfoCodec, err = pool.getFixedIP(ctx, ip, nil); err != nil {
if ipInfoCodec, err = pool.GetFixedIP(ctx, ip, nil); err != nil {
return err
}
if ipInfoCodec.IPInfo.Status.Match(types.IPStatusInUse) {
Expand Down Expand Up @@ -101,7 +102,7 @@ func (pool fixedIPPool) UnassignFixedIP(ctx context.Context, ip types.IP) error
err error
)

if ipInfoCodec, err = pool.getFixedIP(ctx, ip, nil); err != nil {
if ipInfoCodec, err = pool.GetFixedIP(ctx, ip, nil); err != nil {
return err
}

Expand Down Expand Up @@ -130,7 +131,7 @@ func (pool fixedIPPool) BorrowFixedIP(ctx context.Context, ip types.IP, containe

cnt := 0
for cnt < retryMaxCount {
codec, err := pool.getFixedIP(ctx, ip, nil)
codec, err := pool.GetFixedIP(ctx, ip, nil)
if err != nil {
return err
}
Expand All @@ -157,7 +158,7 @@ func (pool fixedIPPool) ReturnFixedIP(ctx context.Context, ip types.IP, containe

cnt := 0
for cnt < retryMaxCount {
codec, err := pool.getFixedIP(ctx, ip, nil)
codec, err := pool.GetFixedIP(ctx, ip, nil)
if err != nil {
return err
}
Expand Down Expand Up @@ -210,7 +211,7 @@ func (pool fixedIPPool) UnallocFixedIP(ctx context.Context, ip types.IP, force b
// return types.ErrFixedIPNotAllocated
// }

if ipInfoCodec, err = pool.getFixedIP(ctx, ip, nil); err != nil {
if ipInfoCodec, err = pool.GetFixedIP(ctx, ip, nil); err != nil {
return err
}

Expand Down Expand Up @@ -253,7 +254,7 @@ func (pool fixedIPPool) logger(method string) *log.Entry {
return log.WithField("Receiver", "fixedIPPool").WithField("Method", method)
}

func (pool fixedIPPool) getFixedIP(
func (pool fixedIPPool) GetFixedIP(
ctx context.Context,
ip types.IP,
allocateFixedIP func(context.Context, types.IP, *codecs.IPInfoCodec) error,
Expand Down Expand Up @@ -311,7 +312,7 @@ func (alloc fixedIPAllocator) AllocFixedIP(ctx context.Context, ip types.IP) err
ipInfoCodec *codecs.IPInfoCodec
err error
)
if ipInfoCodec, err = alloc.getFixedIP(ctx, ip, alloc.createFixedIP); err != nil {
if ipInfoCodec, err = alloc.GetFixedIP(ctx, ip, alloc.createFixedIP); err != nil {
return err
}

Expand Down

0 comments on commit 624d490

Please sign in to comment.