etcd+flanneld不同节点分配到相同网段
环境信息： etcd 3.2.12flannel v0.11OS Centos 7.6 flanneld为节点分配网段策略从subnet代码库里面可以看到，当前flannel支持从etcd和kube api存储网段信息，这里在etcd里面存储，因此只分析在etcd作为网段存储时，地址分配流程
首先flannel启动时候，会获取本机的网络接口，iface或者regex未指定的话则会返回第一个接口，根据这个接口的IP信息来分配网段，首先从源码来看下具体的流程 flannelFlags.Var(&opts.iface, "iface", "interface to use (IP or name) for inter-host communication. Can be specified multiple times to check each option in order. Returns the first match found.") # 第一步，获取本机接口信息 # 如果 iface 和 iface-regex都未指定,则会返回LookupExtIface函数获取到的第一个接口 if len(opts.iface) == 0 && len(opts.ifaceRegex) == 0 { extIface, err = LookupExtIface("", "") if err != nil { log.Error("Failed to find any valid interface to use: ", err) os.Exit(1) } } # LookupExtIface函数定义 func LookupExtIface(ifname string, ifregex string) (*backend.ExternalInterface, error) { var iface *net.Interface var ifaceAddr net.IP var err error if len(ifname) > 0 { if ifaceAddr = net.ParseIP(ifname); ifaceAddr != nil { log.Infof("Searching for interface using %s", ifaceAddr) iface, err = ip.GetInterfaceByIP(ifaceAddr) if err != nil { return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err) } } else { iface, err = net.InterfaceByName(ifname) if err != nil { return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err) } } } else if len(ifregex) > 0 { // Use the regex if specified and the iface option for matching a specific ip or name is not used ifaces, err := net.Interfaces() if err != nil { return nil, fmt.Errorf("error listing all interfaces: %s", err) } // Check IP for _, ifaceToMatch := range ifaces { ifaceIP, err := ip.GetIfaceIP4Addr(&ifaceToMatch) if err != nil { // Skip if there is no IPv4 address continue } matched, err := regexp.MatchString(ifregex, ifaceIP.String()) if err != nil { return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceIP.String()) } if matched { ifaceAddr = ifaceIP iface = &ifaceToMatch break } } // Check Name if iface == nil && ifaceAddr == nil { for _, ifaceToMatch := range ifaces { matched, err := regexp.MatchString(ifregex, ifaceToMatch.Name) if err != nil { return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceToMatch.Name) } if matched { iface = &ifaceToMatch break } } } // Check that nothing was matched if iface == nil { var availableFaces []string for _, f := range ifaces { ip, _ := ip.GetIfaceIP4Addr(&f) // We can safely ignore errors. We just won't log any ip availableFaces = append(availableFaces, fmt.Sprintf("%s:%s", f.Name, ip)) } return nil, fmt.Errorf("Could not match pattern %s to any of the available network interfaces (%s)", ifregex, strings.Join(availableFaces, ", ")) } } else { log.Info("Determining IP address of default interface") if iface, err = ip.GetDefaultGatewayIface(); err != nil { return nil, fmt.Errorf("failed to get default interface: %s", err) } } if ifaceAddr == nil { ifaceAddr, err = ip.GetIfaceIP4Addr(iface) if err != nil { return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name) } } log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr) if iface.MTU == 0 { return nil, fmt.Errorf("failed to determine MTU for %s interface", ifaceAddr) } var extAddr net.IP if len(opts.publicIP) > 0 { extAddr = net.ParseIP(opts.publicIP) if extAddr == nil { return nil, fmt.Errorf("invalid public IP address: %s", opts.publicIP) } log.Infof("Using %s as external address", extAddr) } if extAddr == nil { log.Infof("Defaulting external address to interface address (%s)", ifaceAddr) extAddr = ifaceAddr } # 上述条件都不符合的话会返回下面的值 return &backend.ExternalInterface{ Iface: iface, IfaceAddr: ifaceAddr, ExtAddr: extAddr, }, nil } # 看下ExternalInterface定义（本机全部网络接口的值） type ExternalInterface struct { Iface *net.Interface IfaceAddr net.IP ExtAddr net.IP } # 第二步，分配subnet func (esr *etcdSubnetRegistry) createSubnet(ctx context.Context, sn ip.IP4Net, attrs *LeaseAttrs, ttl time.Duration) (time.Time, error) { key := path.Join(esr.etcdCfg.Prefix, "subnets", MakeSubnetKey(sn)) value, err := json.Marshal(attrs) if err != nil { return time.Time{}, err } # 可以看到会从etcd指定路径下获取subnets的信息,调用MakeSubnetKey函数如下： func MakeSubnetKey(sn ip.IP4Net) string { return sn.StringSep(".", "-") } # 此步会调用StringSep函数，如下： func (ip IP4) StringSep(sep string) string { a, b, c, d := ip.Octets() return fmt.Sprintf("%d%s%d%s%d%s%d", a, sep, b, sep, c, sep, d) } # 会根据第一个字段ip去重，这个就是为什么不同节点会分配到相同的网段信息原因
环境为vbox虚拟机，vbox虚拟机默认的eth0地址10.0.2.15，不同的vbox虚拟机都会有这个地址，而iface又未特别指定，所以会出现不同节点分配到不同IP的情况，
既然原因已经查明，解决办法就有了，步骤如下：
修改flanneld启动文件参数，指定iface ExecStart=/usr/local/bin/flanneld -iface=eth1 \ -etcd-cafile=/etc/kubernetes/ssl/ca.pem \ -etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \ -etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \ -etcd-endpoints=https://10.0.0.111:2379,https://10.0.0.112:2379,https://10.0.0.113:2379 \ -etcd-prefix=/kubernetes/network 删除flannel已经生成的网络配置文件 rm -rf /run/flannel/* reload和重启flannel服务 # 使用systemd管理的服务执行如下，不用systemd的可以按自己的方式来 systemctl daemon-reload systemctl restart flanneld.service 查看验证 # 命令行查看 etcdctl --endpoints=https://10.0.0.111:2379,https://10.0.0.112:2379,https://10.0.0.113:2379 --ca-file=/etc/kubernetes/ssl/ca.pem --cert-file=/etc/kubernetes/ssl/kubernetes.pem --key-file=/etc/kubernetes/ssl/kubernetes-key.pem ls /kubernetes/network/subnets # 结果如下 /kubernetes/network/subnets/10.1.100.0-24 /kubernetes/network/subnets/10.1.26.0-24 /kubernetes/network/subnets/10.1.11.0-24 /kubernetes/network/subnets/10.1.77.0-24