kubelet: Set --config to avoid using legacy defaults

Fixes #12310
It also help avoid similar issues as #12164 in the future. I.e. we do not need to explicity override defaults by using cli flags.

Signed-off-by: Boleyn Su <[email protected]>

Author: BoleynSu

pkg/agent/config/config.go

@@ -601,6 +601,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		return nil, err
 	}
 
+	kubeletConfig := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf")
 	// Ensure kubelet config dir exists
 	kubeletConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf.d")
 	if err := os.MkdirAll(kubeletConfigDir, 0700); err != nil {
@@ -635,6 +636,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
 	nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
 	nodeConfig.AgentConfig.ClientCA = clientCAFile
+	nodeConfig.AgentConfig.KubeletConfig = kubeletConfig
 	nodeConfig.AgentConfig.KubeletConfigDir = kubeletConfigDir
 	nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
 	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy

pkg/daemons/agent/agent.go

@@ -64,16 +64,20 @@ func startKubeProxy(ctx context.Context, cfg *daemonconfig.Agent) error {
 func startKubelet(ctx context.Context, cfg *daemonconfig.Agent) error {
 	argsMap, defaultConfig, err := kubeletArgsAndConfig(cfg)
 	if err != nil {
-		return pkgerrors.WithMessage(err, "prepare default configuration drop-in")
+		return pkgerrors.WithMessage(err, "prepare default kubelet configuration")
 	}
 
-	extraArgs, err := extractConfigArgs(cfg.KubeletConfigDir, cfg.ExtraKubeletArgs, defaultConfig)
-	if err != nil {
-		return pkgerrors.WithMessage(err, "prepare user configuration drop-ins")
+	if err := writeKubeletConfig(cfg.KubeletConfig, defaultConfig); err != nil {
+		return pkgerrors.WithMessage(err, "generate default kubelet configuration")
 	}
 
-	if err := writeKubeletConfig(cfg.KubeletConfigDir, defaultConfig); err != nil {
-		return pkgerrors.WithMessage(err, "generate default kubelet configuration drop-in")
+	// Remove default kubelet configuration generated in drop-in directory
+	// TODO: Clean this up some time after k3s with version <=1.33 is EOL.
+	os.Remove(filepath.Join(cfg.KubeletConfigDir, "00-"+version.Program+"-defaults.conf"))
+
+	extraArgs, err := extractConfigArgs(cfg.KubeletConfigDir, cfg.ExtraKubeletArgs)
+	if err != nil {
+		return pkgerrors.WithMessage(err, "prepare user kubelet configuration drop-ins")
 	}
 
 	args := util.GetArgs(argsMap, extraArgs)
@@ -99,7 +103,7 @@ func ImageCredProvAvailable(cfg *daemonconfig.Agent) bool {
 // extractConfigArgs strips out any --config or --config-dir flags from the
 // provided args list, and if set, copies the content of the file or dir into
 // the target drop-in directory.
-func extractConfigArgs(path string, extraArgs []string, config *kubeletconfig.KubeletConfiguration) ([]string, error) {
+func extractConfigArgs(path string, extraArgs []string) ([]string, error) {
 	args := make([]string, 0, len(extraArgs))
 	strippedArgs := map[string]string{}
 	var skipVal bool
@@ -158,7 +162,7 @@ func writeKubeletConfig(path string, config *kubeletconfig.KubeletConfiguration)
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(filepath.Join(path, "00-"+version.Program+"-defaults.conf"), b, 0600)
+	return os.WriteFile(path, b, 0600)
 }
 
 func defaultKubeletConfig(cfg *daemonconfig.Agent) (*kubeletconfig.KubeletConfiguration, error) {

pkg/daemons/agent/agent_linux.go

@@ -75,11 +75,9 @@ func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.
 		return nil, nil, err
 	}
 	argsMap := map[string]string{
+		"config": cfg.KubeletConfig,
 		"config-dir": cfg.KubeletConfigDir,
 		"kubeconfig": cfg.KubeConfigKubelet,
-		// note: KubeletConfiguration will omit this field when marshalling if it is set to 0, so we set it via CLI
-		// https://github.com/k3s-io/k3s/issues/12164
-		"read-only-port": "0",
 	}
 
 	if cfg.RootDir != "" {

pkg/daemons/agent/agent_windows.go

@@ -48,11 +48,9 @@ func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.
 		return nil, nil, err
 	}
 	argsMap := map[string]string{
+		"config": cfg.KubeletConfig,
 		"config-dir": cfg.KubeletConfigDir,
 		"kubeconfig": cfg.KubeConfigKubelet,
-		// note: KubeletConfiguration will omit this field when marshalling if it is set to 0, so we set it via CLI
-		// https://github.com/k3s-io/k3s/issues/12164
-		"read-only-port": "0",
 	}
 	if cfg.RootDir != "" {
 		argsMap["root-dir"] = cfg.RootDir

pkg/daemons/config/types.go

@@ -119,6 +119,7 @@ type Agent struct {
 	ClusterDomain           string
 	ResolvConf              string
 	RootDir                 string
+	KubeletConfig           string
 	KubeletConfigDir        string
 	KubeConfigKubelet       string
 	KubeConfigKubeProxy     string