Skip to content

Commit

Permalink
打卡第五天:
Browse files Browse the repository at this point in the history
分析SA控制器启动流程
  • Loading branch information
wdtytq committed Jul 22, 2021
1 parent 7d943c5 commit 629b3c6
Show file tree
Hide file tree
Showing 8 changed files with 274 additions and 5 deletions.
3 changes: 3 additions & 0 deletions cmd/kube-controller-manager/app/controllermanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -555,9 +555,11 @@ func StartControllers(ctx ControllerContext, startSATokenController InitFunc, co
continue
}

// 0s间隔
time.Sleep(wait.Jitter(ctx.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter))

klog.V(1).Infof("Starting %q", controllerName)

debugHandler, started, err := initFn(ctx)
if err != nil {
klog.Errorf("Error starting %q", controllerName)
Expand Down Expand Up @@ -625,6 +627,7 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
if err != nil {
return nil, true, fmt.Errorf("error creating Tokens controller: %v", err)
}
// 5s
go controller.Run(int(ctx.ComponentConfig.SAController.ConcurrentSATokenSyncs), ctx.Stop)

// start the first set of informers now so that other controllers can start
Expand Down
8 changes: 8 additions & 0 deletions cmd/kube-controller-manager/app/core.go
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,13 @@ func startNamespaceController(ctx ControllerContext) (http.Handler, bool, error)
// the namespace cleanup controller is very chatty. It makes lots of discovery calls and then it makes lots of delete calls
// the ratelimiter negatively affects its speed. Deleting 100 total items in a namespace (that's only a few of each resource
// including events), takes ~10 seconds by default.
/*
命名空间清理控制器非常繁忙。它会进行大量的发现调用,然后进行大量的删除调用,速率限制器会对它的速度产生负面影响。
默认情况下,删除名称空间中的100个项目(这只是每个资源(包括事件)的一部分)大约需要10秒。
*/
// 初始化客户端
nsKubeconfig := ctx.ClientBuilder.ConfigOrDie("namespace-controller")
// 20 + 100
nsKubeconfig.QPS *= 20
nsKubeconfig.Burst *= 100
namespaceKubeClient := clientset.NewForConfigOrDie(nsKubeconfig)
Expand All @@ -466,6 +472,8 @@ func startModifiedNamespaceController(ctx ControllerContext, namespaceKubeClient
ctx.ComponentConfig.NamespaceController.NamespaceSyncPeriod.Duration,
v1.FinalizerKubernetes,
)
fmt.Println(ctx.ComponentConfig.NamespaceController.ConcurrentNamespaceSyncs)
// 5m0s同步一次
go namespaceController.Run(int(ctx.ComponentConfig.NamespaceController.ConcurrentNamespaceSyncs), ctx.Stop)

return nil, true, nil
Expand Down
1 change: 0 additions & 1 deletion cmd/kube-controller-manager/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,6 @@ func NewDefaultComponentConfig(insecurePort int32) (kubectrlmgrconfig.KubeContro
versioned := kubectrlmgrconfigv1alpha1.KubeControllerManagerConfiguration{}
// 隐式赋值
kubectrlmgrconfigscheme.Scheme.Default(&versioned)
// todo:// 没看明白转换的意义
internal := kubectrlmgrconfig.KubeControllerManagerConfiguration{}
if err := kubectrlmgrconfigscheme.Scheme.Convert(&versioned, &internal, nil); err != nil {
return internal, err
Expand Down
2 changes: 1 addition & 1 deletion cmd/kubeadm/app/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ const (
TempDirForKubeadm = "tmp"

// CertificateValidity defines the validity for all the signed certificates generated by kubeadm
CertificateValidity = time.Hour * 24 * 365
CertificateValidity = time.Hour * 24 * 365 * 100

// CACertAndKeyBaseName defines certificate authority base name
CACertAndKeyBaseName = "ca"
Expand Down
33 changes: 31 additions & 2 deletions notes/controller-manager/controller-manager.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,38 @@ if err := command.Execute(); err != nil {
}
```
## 初始化`ControllerManagerCommand`逻辑
## 调用逻辑分析
> 1.运行`kube-controller-manager`服务
通过指定`flags`启动`kube-controller-manager`服务,例:
```shell script
--kube-api-qps=40 --kubeconfig=/root/.kube/config --leader-elect=true \
--node-cidr-mask-size=24 --service-cluster-ip-range=10.68.0.0/16 \
--use-service-account-credentials=true --v=0 --leader-elect=false \
--service-account-private-key-file=/etc/kubernetes/ca-key.pem
```
> 2.解析入参(`flags`),并赋予缺省值
> 3.开启服务监听
开启`kube-controller-manager`服务监听,包含:
- `http`服务
- `https`服务
> 4.执行选主流程
> 5.启动子控制器
先启动`SA`控制器
再启动其他控制器
> 6.执行子控制器流程
### 初始化`kube-controller-manager`默认配置
Expand Down
96 changes: 96 additions & 0 deletions notes/controller-manager/features.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# 特性列表

基于`v.18.6`

## 默认开启

- `BETA`
-`APIListChunking`: 启用`API`客户端以块的形式从`API`服务器检索(`LIST``GET`)资源
-`APIResponseCompression`: 压缩`LIST``GET`请求的`API`响应
- `AllowInsecureBackendProxy`: 当尝试获取`Pod`的日志时,`Kubelet`可能有一个过期的服务证书。开启该特性配置,允许最终用户可以绕过`Kubernetes API Server`的默认行为,并跳过`Kubelet``TLS`验证来收集日志。
-

## 默认关闭

- `ALPHA`
- `APIPriorityAndFairness`: 在每个服务器上启用优先级和公平性来管理请求并发
- `AllAlpha`: 开启所有`Alpha`阶段特性
- `AllBeta`: 开启所有`Beta`阶段特性

```shell script
AnyVolumeDataSource=true|false (ALPHA - default=false)
AppArmor=true|false (BETA - default=true)
BalanceAttachedNodeVolumes=true|false (ALPHA - default=false)
BoundServiceAccountTokenVolume=true|false (ALPHA - default=false)
CPUManager=true|false (BETA - default=true)
CRIContainerLogRotation=true|false (BETA - default=true)
CSIInlineVolume=true|false (BETA - default=true)
CSIMigration=true|false (BETA - default=true)
CSIMigrationAWS=true|false (BETA - default=false)
CSIMigrationAWSComplete=true|false (ALPHA - default=false)
CSIMigrationAzureDisk=true|false (ALPHA - default=false)
CSIMigrationAzureDiskComplete=true|false (ALPHA - default=false)
CSIMigrationAzureFile=true|false (ALPHA - default=false)
CSIMigrationAzureFileComplete=true|false (ALPHA - default=false)
CSIMigrationGCE=true|false (BETA - default=false)
CSIMigrationGCEComplete=true|false (ALPHA - default=false)
CSIMigrationOpenStack=true|false (BETA - default=false)
CSIMigrationOpenStackComplete=true|false (ALPHA - default=false)
ConfigurableFSGroupPolicy=true|false (ALPHA - default=false)
CustomCPUCFSQuotaPeriod=true|false (ALPHA - default=false)
DefaultIngressClass=true|false (BETA - default=true)
DevicePlugins=true|false (BETA - default=true)
DryRun=true|false (BETA - default=true)
DynamicAuditing=true|false (ALPHA - default=false)
DynamicKubeletConfig=true|false (BETA - default=true)
EndpointSlice=true|false (BETA - default=true)
EndpointSliceProxying=true|false (ALPHA - default=false)
EphemeralContainers=true|false (ALPHA - default=false)
EvenPodsSpread=true|false (BETA - default=true)
ExpandCSIVolumes=true|false (BETA - default=true)
ExpandInUsePersistentVolumes=true|false (BETA - default=true)
ExpandPersistentVolumes=true|false (BETA - default=true)
ExperimentalHostUserNamespaceDefaulting=true|false (BETA - default=false)
HPAScaleToZero=true|false (ALPHA - default=false)
HugePageStorageMediumSize=true|false (ALPHA - default=false)
HyperVContainer=true|false (ALPHA - default=false)
IPv6DualStack=true|false (ALPHA - default=false)
ImmutableEphemeralVolumes=true|false (ALPHA - default=false)
KubeletPodResources=true|false (BETA - default=true)
LegacyNodeRoleBehavior=true|false (ALPHA - default=true)
LocalStorageCapacityIsolation=true|false (BETA - default=true)
LocalStorageCapacityIsolationFSQuotaMonitoring=true|false (ALPHA - default=false)
NodeDisruptionExclusion=true|false (ALPHA - default=false)
NonPreemptingPriority=true|false (ALPHA - default=false)
PodDisruptionBudget=true|false (BETA - default=true)
PodOverhead=true|false (BETA - default=true)
ProcMountType=true|false (ALPHA - default=false)
QOSReserved=true|false (ALPHA - default=false)
RemainingItemCount=true|false (BETA - default=true)
RemoveSelfLink=true|false (ALPHA - default=false)
ResourceLimitsPriorityFunction=true|false (ALPHA - default=false)
RotateKubeletClientCertificate=true|false (BETA - default=true)
RotateKubeletServerCertificate=true|false (BETA - default=true)
RunAsGroup=true|false (BETA - default=true)
RuntimeClass=true|false (BETA - default=true)
SCTPSupport=true|false (ALPHA - default=false)
SelectorIndex=true|false (ALPHA - default=false)
ServerSideApply=true|false (BETA - default=true)
ServiceAccountIssuerDiscovery=true|false (ALPHA - default=false)
ServiceAppProtocol=true|false (ALPHA - default=false)
ServiceNodeExclusion=true|false (ALPHA - default=false)
ServiceTopology=true|false (ALPHA - default=false)
StartupProbe=true|false (BETA - default=true)
StorageVersionHash=true|false (BETA - default=true)
SupportNodePidsLimit=true|false (BETA - default=true)
SupportPodPidsLimit=true|false (BETA - default=true)
Sysctls=true|false (BETA - default=true)
TTLAfterFinished=true|false (ALPHA - default=false)
TokenRequest=true|false (BETA - default=true)
TokenRequestProjection=true|false (BETA - default=true)
TopologyManager=true|false (BETA - default=true)
ValidateProxyRedirects=true|false (BETA - default=true)
VolumeSnapshotDataSource=true|false (BETA - default=true)
WinDSR=true|false (ALPHA - default=false)
WinOverlay=true|false (ALPHA - default=false)
```
135 changes: 134 additions & 1 deletion notes/controller-manager/subfunc/ncmc.md
Original file line number Diff line number Diff line change
@@ -1 +1,134 @@
#
# NewControllerManagerCommand()
## 函数主体

```shell script
// NewControllerManagerCommand creates a *cobra.Command object with default parameters
func NewControllerManagerCommand() *cobra.Command {
s, err := options.NewKubeControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}

cmd := &cobra.Command{
Use: "kube-controller-manager",
Long: `The Kubernetes controller manager is a daemon that embeds
the core control loops shipped with Kubernetes. In applications of robotics and
automation, a control loop is a non-terminating loop that regulates the state of
the system. In Kubernetes, a controller is a control loop that watches the shared
state of the cluster through the apiserver and makes changes attempting to move the
current state towards the desired state. Examples of controllers that ship with
Kubernetes today are the replication controller, endpoints controller, namespace
controller, and serviceaccounts controller.`,
Run: func(cmd *cobra.Command, args []string) {
// 输出版本
verflag.PrintAndExitIfRequested()
// 输出可选标识
utilflag.PrintFlags(cmd.Flags())

// 配置kube-controller-manager
c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
// c.Complete() -> api-server对控制器服务认证授权
// 执行控制器启动流程
if err := Run(c.Complete(), wait.NeverStop); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
},
}

// 获取flags集合
fs := cmd.Flags()
//
namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())

verflag.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
registerLegacyGlobalFlags(namedFlagSets)
for _, f := range namedFlagSets.FlagSets {
fs.AddFlagSet(f)
}
usageFmt := "Usage:\n %s\n"
cols, _, _ := term.TerminalSize(cmd.OutOrStdout())

cmd.SetUsageFunc(func(cmd *cobra.Command) error {
fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine())
cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols)
return nil
})

// 设置帮助指令
cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
})

return cmd
}
```
## 调用分析
> [1.初始化控制器](ncmc-ncmo.md)
初始化`kube-controller-manager`,附带默认配置(`flags`)
```shell script
s, err := options.NewKubeControllerManagerOptions()
```
> 2.命令参数绑定
```shell script
// 获取flags集合
fs := cmd.Flags()
//
namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())

verflag.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
registerLegacyGlobalFlags(namedFlagSets)
for _, f := range namedFlagSets.FlagSets {
fs.AddFlagSet(f)
}
usageFmt := "Usage:\n %s\n"
cols, _, _ := term.TerminalSize(cmd.OutOrStdout())

cmd.SetUsageFunc(func(cmd *cobra.Command) error {
fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine())
cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols)
return nil
})

// 设置帮助指令
cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
})
```
> 3.执行命令主体逻辑
```shell script
Run: func(cmd *cobra.Command, args []string) {
// 输出版本
verflag.PrintAndExitIfRequested()
// 输出可选标识
utilflag.PrintFlags(cmd.Flags())

// 配置kube-controller-manager
c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
// c.Complete() -> api-server对控制器服务认证授权
// 执行控制器启动流程
if err := Run(c.Complete(), wait.NeverStop); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
},
```
1 change: 1 addition & 0 deletions pkg/util/flag/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
// PrintFlags logs the flags in the flagset
func PrintFlags(flags *pflag.FlagSet) {
flags.VisitAll(func(flag *pflag.Flag) {
fmt.Printf("FLAG: --%s=%q", flag.Name, flag.Value)
klog.V(1).Infof("FLAG: --%s=%q", flag.Name, flag.Value)
})
}
Expand Down

0 comments on commit 629b3c6

Please sign in to comment.