kubernetes 源码分析之kubeadm（二）

上一篇通过kubeadm去部署kubernetes集群。这篇进入代码进行讲解。先看kubeadm init这个创建master的命令是怎样运行的cmd/kubeadm/app/cmd/cmd.go。

    cmds.AddCommand(NewCmdCompletion(out, ""))
    cmds.AddCommand(NewCmdInit(out))
    cmds.AddCommand(NewCmdJoin(out))
    cmds.AddCommand(NewCmdReset(out))
    cmds.AddCommand(NewCmdVersion(out))
    cmds.AddCommand(NewCmdToken(out, err))

注册了这些方法，先看init方法cmd/kubeadm/app/cmd/init.go

    cmd := &cobra.Command{
        Use:   "init",
        Short: "Run this in order to set up the Kubernetes master",
        Run: func(cmd *cobra.Command, args []string) {
            api.Scheme.Default(cfg)
            internalcfg := &kubeadmapi.MasterConfiguration{}
            api.Scheme.Convert(cfg, internalcfg, nil)

            i, err := NewInit(cfgPath, internalcfg, skipPreFlight)
            kubeadmutil.CheckErr(err)
            kubeadmutil.CheckErr(i.Validate())
            kubeadmutil.CheckErr(i.Run(out))
        },
    }

运行init的时候就是执行了上面的Run方法。这个方法先是配置参数然后执行i.Run(out)，进入看看配置参数NewInit这个方法

func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight bool) (*Init, error) {

    fmt.Println("[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.")

    if cfgPath != "" {
        b, err := ioutil.ReadFile(cfgPath)
        if err != nil {
            return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
        }
        if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), b, cfg); err != nil {
            return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
        }
    }

    // Set defaults dynamically that the API group defaulting can't (by fetching information from the internet, looking up network interfaces, etc.)
    err := setInitDynamicDefaults(cfg)
    if err != nil {
        return nil, err
    }

    if !skipPreFlight {
        fmt.Println("[preflight] Running pre-flight checks")

        // First, check if we're root separately from the other preflight checks and fail fast
        if err := preflight.RunRootCheckOnly(); err != nil {
            return nil, err
        }

        // Then continue with the others...
        if err := preflight.RunInitMasterChecks(cfg); err != nil {
            return nil, err
        }
    } else {
        fmt.Println("[preflight] Skipping pre-flight checks")
    }

    // Try to start the kubelet service in case it's inactive
    preflight.TryStartKubelet()

    return &Init{cfg: cfg}, nil
}

这个方法先读取配置文件，然后就进行环境监测，当然你可以按照上一篇介绍的跳过，进入RunInitMasterChecks看看到底监测什么东西：

    checks := []Checker{
        SystemVerificationCheck{},
        IsRootCheck{},
        HostnameCheck{},
        ServiceCheck{Service: "kubelet", CheckIfActive: false},
        ServiceCheck{Service: "docker", CheckIfActive: true},
        FirewalldCheck{ports: []int{int(cfg.API.BindPort), 10250}},
        PortOpenCheck{port: int(cfg.API.BindPort)},
        PortOpenCheck{port: 10250},
        PortOpenCheck{port: 10251},
        PortOpenCheck{port: 10252},
        HTTPProxyCheck{Proto: "https", Host: cfg.API.AdvertiseAddress, Port: int(cfg.API.BindPort)},
        DirAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
        DirAvailableCheck{Path: "/var/lib/kubelet"},
        FileContentCheck{Path: bridgenf, Content: []byte{'1'}},
        InPathCheck{executable: "ip", mandatory: true},
        InPathCheck{executable: "iptables", mandatory: true},
        InPathCheck{executable: "mount", mandatory: true},
        InPathCheck{executable: "nsenter", mandatory: true},
        InPathCheck{executable: "ebtables", mandatory: false},
        InPathCheck{executable: "ethtool", mandatory: false},
        InPathCheck{executable: "socat", mandatory: false},
        InPathCheck{executable: "tc", mandatory: false},
        InPathCheck{executable: "touch", mandatory: false},
    }

上面截取监测的对象，主要是一些权限、端口、文件和安装包监测。这些监测通过后就启动kubelet，所以上一篇的启动kubelet其实可以省略。参数配置完成接下来就是运行了

// Run executes master node provisioning, including certificates, needed static pod manifests, etc.
func (i *Init) Run(out io.Writer) error {

    // PHASE 1: Generate certificates
    err := certphase.CreatePKIAssets(i.cfg)
    if err != nil {
        return err
    }

    // PHASE 2: Generate kubeconfig files for the admin and the kubelet

    masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddress, i.cfg.API.BindPort)
    err = kubeconfigphase.CreateInitKubeConfigFiles(masterEndpoint, i.cfg.CertificatesDir, kubeadmapi.GlobalEnvParams.KubernetesDir)
    if err != nil {
        return err
    }

    // PHASE 3: Bootstrap the control plane
    if err := kubemaster.WriteStaticPodManifests(i.cfg); err != nil {
        return err
    }

    adminKubeConfigPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)
    client, err := kubemaster.CreateClientAndWaitForAPI(adminKubeConfigPath)
    if err != nil {
        return err
    }

    if err := apiconfigphase.UpdateMasterRoleLabelsAndTaints(client); err != nil {
        return err
    }

    // Is deployment type self-hosted?
    if i.cfg.SelfHosted {
        // Temporary control plane is up, now we create our self hosted control
        // plane components and remove the static manifests:
        fmt.Println("[self-hosted] Creating self-hosted control plane...")
        if err := kubemaster.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
            return err
        }
    }

    // PHASE 4: Set up the bootstrap tokens
    fmt.Printf("[token] Using token: %s\n", i.cfg.Token)

    tokenDescription := "The default bootstrap token generated by 'kubeadm init'."
    if err := tokenphase.UpdateOrCreateToken(client, i.cfg.Token, false, i.cfg.TokenTTL, kubeadmconstants.DefaultTokenUsages, tokenDescription); err != nil {
        return err
    }

    if err := tokenphase.CreateBootstrapConfigMap(adminKubeConfigPath); err != nil {
        return err
    }

    // PHASE 5: Install and deploy all addons, and configure things as necessary

    // Create the necessary ServiceAccounts
    err = apiconfigphase.CreateServiceAccounts(client)
    if err != nil {
        return err
    }

    err = apiconfigphase.CreateRBACRules(client)
    if err != nil {
        return err
    }

    if err := addonsphase.CreateEssentialAddons(i.cfg, client); err != nil {
        return err
    }

    ctx := map[string]string{
        "KubeConfigPath": path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName),
        "KubeConfigName": kubeadmconstants.AdminKubeConfigFileName,
        "Token":          i.cfg.Token,
        "MasterIP":       i.cfg.API.AdvertiseAddress,
        "MasterPort":     strconv.Itoa(int(i.cfg.API.BindPort)),
    }

    return initDoneTempl.Execute(out, ctx)
}

这个init.go里面的Run的方法分为5个步骤：
1.生成证书如ca.cert、ca.key等，证书在/etc/kubernetes/pki下面：

apiserver.crt  apiserver-kubelet-client.crt  ca.crt  front-proxy-ca.crt  front-proxy-client.crt  sa.key
apiserver.key  apiserver-kubelet-client.key  ca.key  front-proxy-ca.key  front-proxy-client.key  sa.pub

2.生成kubeconfig配置文件，/etc/kubernetes

admin.conf  controller-manager.conf kubelet.conf  scheduler.conf

3.生成Manifests，在/etc/kubernetes/manifests

etcd.yaml  kube-apiserver.yaml  kube-controller-manager.yaml  kube-scheduler.yaml

4.创建更新token
5.启动addon插件如proxy、dns等
其中有几个技术点需要说明，第三步生成的Manifests启动master组件，这个涉及到kubelet通过manifest文件启动容器，我之前在kubelet源码分析里面有讲解，在次不再赘述。第五步，会通过daemonset启动proxy，代码如下：

proxyDaemonSetBytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet, struct{ Image, ClusterCIDR, MasterTaintKey string }{
        Image:          images.GetCoreImage("proxy", cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
        ClusterCIDR:    getClusterCIDR(cfg.Networking.PodSubnet),
        MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster,
    })
    if err != nil {
        return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)
    }

    dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment, struct{ ImageRepository, Arch, Version, DNSDomain, MasterTaintKey string }{
        ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix,
        Arch:            runtime.GOARCH,
        Version:         KubeDNSVersion,
        DNSDomain:       cfg.Networking.DNSDomain,
        MasterTaintKey:  kubeadmconstants.LabelNodeRoleMaster,
    })

启动两个主要组件kube-proxy和kubedns。好了，这个master已经讲解完毕。后面的blog讲解jion等其他东西。